From 4e3630e7d1b5dfbc0d145361418e8857022a2d65 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Sep 2018 18:41:39 +0100 Subject: [PATCH 0001/2779] updated tests/test_anonymize.sh --- tests/test_anonymize.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 116cfad8f..ef1d02e69 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -53,14 +53,19 @@ if [ -z "$test_nums" ]; then fi hr - echo "testing --email replacememnt format:" run_grep "@" $anonymize --email <<< "hari@domain.com" run_grep "@" $anonymize -E <<< "hari@domain.com" - echo "testing --ip-prefix behaviour:" + run_grep ".1" $anonymize -a --ip-prefix <<< "4.3.2.1" + run_grep ".1/" $anonymize --ip-prefix <<< "4.3.2.1/24" run_grep ".1" $anonymize --ip-prefix <<< "4.3.2.1" + run_grep ".4" $anonymize --ip-prefix <<< "ip-1-2-3-4" + run_grep "ip-1-2-3-4-5" $anonymize --ip-prefix <<< "ip-1-2-3-4-5" + run_grep "dip-1-2-3-4" $anonymize --ip-prefix <<< "dip-1-2-3-4" + run_grep "5.4.3.2.1" $anonymize --ip-prefix <<< "5.4.3.2.1" + run_grep "log4j-1.2.3.4.jar" $anonymize --ip-prefix <<< "log4j-1.2.3.4.jar" + run_grep "/usr/hdp/2.6.2.0-123" $anonymize --ip-prefix <<< "/usr/hdp/2.6.2.0-123" - echo "testing --hash-hostnames:" run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path" run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir' run_grep '-host [a-f0-9]{12}' $anonymize --hash-hostnames <<< '-host blah' @@ -340,8 +345,8 @@ dest[84]="ldap:///dc=,dc=??sub?(givenName=)" src[85]="ip-1-2-3-4-5" dest[85]="ip-1-2-3-4-5" -src[86]="dip-1-2-3-4-5" -dest[86]="dip-1-2-3-4-5" +src[86]="dip-1-2-3-4" +dest[86]="dip-1-2-3-4" src[87]="1.2.3.4.5" dest[87]="1.2.3.4.5" @@ -350,7 +355,7 @@ src[88]="-host blah" dest[88]="-host " src[89]="--hostname=blah --anotherswitch" -dest[89]="-hostname= --anotherswitch" +dest[89]="--hostname= --anotherswitch" src[90]="host=test" dest[90]="host=" @@ -365,6 +370,15 @@ dest[92]="hostname=;port=92" src[93]="hari@domain.com, hari@domain2.co.uk" dest[93]="@, @" +src[94]="spark://hari@192.168.0.1" +dest[94]="spark://@" + +src[95]="/usr/hdp/2.6.2.0-123/blah" +dest[95]="/usr/hdp/2.6.2.0-123/blah" + +src[96]="log4j-1.2.3.4.jar" +dest[96]="log4j-1.2.3.4.jar" + # check escape codes get stripped if present (eg. if piping from grep --color-yes) #src[88]="some^[[01;31m^[[Khost^[[m^[[Kname:443" From b25b6a3d7e16af3c8610e3082dd2dabd819548e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Sep 2018 18:41:43 +0100 Subject: [PATCH 0002/2779] fixed --ip-prefix parsing, improve ip and ip prefix --- anonymize.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/anonymize.py b/anonymize.py index 73d981e5e..1c86d5d1d 100755 --- a/anonymize.py +++ b/anonymize.py @@ -89,7 +89,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.3' +__version__ = '0.8.4' class Anonymize(CLI): @@ -111,8 +111,8 @@ def __init__(self): self.hash_salt = None # order of iteration of application matters because we must do more specific matches before less specific ones self.anonymizations = OrderedDict([ - ('ip', False), ('ip_prefix', False), + ('ip', False), ('subnet_mask', False), ('mac', False), ('kerberos', False), @@ -308,8 +308,13 @@ def __init__(self): sep=arg_sep, pass_word_phrase=pass_word_phrase, pw=password_quoted), + 'ip': r'(?/', 'ip2': r'', 'ip3': r'', - 'ip_prefix': r'.', + 'ip_prefix': r'.\1/', + 'ip_prefix2': r'.\1', + 'ip_prefix3': r'.\1', 'subnet_mask': r'', 'kerberos': r'host/\1@', 'kerberos2': r'host/@', @@ -539,6 +546,9 @@ def process_options(self): if _ == 'ip_prefix': continue self.anonymizations[_] = True + if self.get_opt('ip_prefix'): + self.anonymizations['ip_prefix'] = True + self.anonymizations['ip'] = False else: for _ in self.anonymizations: if _ in ('subnet_mask', 'mac', 'group'): @@ -702,11 +712,6 @@ def prepare_regex(self): r'(?!\(\w+\.java:\d+\))' + \ self.negative_host_lookbehind ) - ip_regex_strict = r'(? Date: Thu, 27 Sep 2018 19:52:14 +0100 Subject: [PATCH 0003/2779] stopped domain/fqdn matching java -Dsome.net.property= and improved custom anchoring, disabled --proxy as it is incompatible with --ip-prefix and --hash-hostnames --- anonymize.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/anonymize.py b/anonymize.py index 1c86d5d1d..9d51a8979 100755 --- a/anonymize.py +++ b/anonymize.py @@ -89,7 +89,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.4' +__version__ = '0.8.5' class Anonymize(CLI): @@ -121,7 +121,6 @@ def __init__(self): ('ldap', False), ('user', False), ('group', False), - ('proxy', False), ('http_auth', False), ('cisco', False), ('screenos', False), @@ -130,6 +129,7 @@ def __init__(self): ('fqdn', False), ('domain', False), ('hostname', False), + #('proxy', False), ('windows', False), ('custom', False), ]) @@ -339,9 +339,8 @@ def __init__(self): .format(ldap_attribs='|'.join(ldap_attributes), ldap_values=ldap_values), 'port': r'{host}:\d+(?!\.?[\w-])'.format(host=host_regex), 'proxy': r'proxy {} port \d+'.format(host_regex), - 'proxy2': 'Trying' + ip_regex, - 'proxy3': 'Connected to ' + host_regex + r'\($ip_regex\) port \d+', - 'proxy4': r'(Via:\s\S+\s)' + ip_regex + '.*', + 'proxy2': 'Connected to ' + host_regex + r'\s*\(' + ip_regex + r'|[^\)]+\) port \d+', + #'proxy3': r'(Via:\s\S+\s)' + ip_regex + '.*', 'http_auth': r'(https?:\/\/)[^:]+:[^\@]*\@', 'http_auth2': r'(Proxy auth using \w+ with user )([\'"]).+([\'"])', 'http_auth3': r'\bAuthorization:\s+Basic\s+[A-Za-z0-9]+', @@ -409,12 +408,11 @@ def __init__(self): 'ldap2': ldap_lambda_lowercase, 'ldap3': ldap_lambda_lowercase, 'proxy': r'proxy port ', - 'proxy2': r'Trying ', - 'proxy3': r'Connected to () port ', + 'proxy2': r'Connected to () port ', + 'proxy3': r'\1', 'http_auth': r'$1:@', 'http_auth2': r'\1\'\2\3/', 'http_auth3': r'Authorization: Basic ', - 'proxy4': r'\1', 'cisco': r'username password ', 'cisco2': r'password ', 'cisco3': r'secret ', @@ -505,9 +503,9 @@ def add_options(self): help='Apply LDAP anonymization (~100 attribs eg. CN, DN, OU, UID, sAMAccountName, memberOf...)') self.add_opt('-E', '--email', action='store_true', help='Apply email format anonymization') - self.add_opt('-x', '--proxy', action='store_true', - help='Apply anonymization to remove proxy host, user etc (eg. from curl -iv output). You ' + \ - 'should probably also apply --ip and --host if using this. Auto enables --http-auth') + #self.add_opt('-x', '--proxy', action='store_true', + # help='Apply anonymization to remove proxy host, user etc (eg. from curl -iv output). You ' + \ + # 'should probably also apply --ip and --host if using this. Auto enables --http-auth') self.add_opt('-N', '--network', action='store_true', help='Apply all network anonymization, whether Cisco, ScreenOS, JunOS for secrets, auth, ' + \ 'usernames, passwords, md5s, PSKs, AS, SNMP etc.') @@ -603,8 +601,8 @@ def _process_options_host(self): self.anonymizations['email'] = True self.anonymizations['domain'] = True self.anonymizations['fqdn'] = True - if self.anonymizations['proxy']: - self.anonymizations['http_auth'] = True + #if self.anonymizations['proxy']: + # self.anonymizations['http_auth'] = True def _process_options_network(self): if self.anonymizations['network']: @@ -661,7 +659,7 @@ def load_file(filename, boundary=False): log.warning('ignoring invalid regex from %s: %s', os.path.basename(filename), line) continue if boundary: - line = r'(\b|[^A-Za-z])' + line + r'(\b|[^A-Za-z])' + line = r'(?:(?<=\b)|(?<=[^A-Za-z]))' + line + r'(?=\b|[^A-Za-z])' regex_list.append(line) raw = '|'.join(regex_list) #log.debug('custom_raw: %s', raw) @@ -697,16 +695,24 @@ def prepare_regex(self): self.negative_host_lookbehind + r':(\d{1,5}(?!\.?\w))', ) self.compile('domain', + # don't match java -some.net.property + #r'(?\2', line) + i += 1 + line = regex.sub(r'', line) #line = re.sub(regex, r'\1\2', line) + log.debug('anonymize_custom: %s => %s', i, line) return line @staticmethod From 451d284d083134154546c6a33217ffa6f80a1afc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Sep 2018 19:55:59 +0100 Subject: [PATCH 0004/2779] updated README.md --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index e9f448a4a..122b68e97 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,18 @@ Environment variables are supported for convenience and also to hide credentials ### DevOps Python Tools ### +- ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - anonymizes: + - hostnames / domains / FQDNs + - email addresses + - IP + MAC addresses + - Kerberos principals + - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) + - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings + - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` + - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) + - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications + - ```--hash-hostnames``` - generates salted hashes that look like Docker temporary container ids so that vendors support teams can differentiate hosts in clusters - [Spark](https://spark.apache.org/) & Data Format Converters: - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter From d03d5aad42b270490ebaac41911644ef261c1f12 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Sep 2018 19:56:12 +0100 Subject: [PATCH 0005/2779] updated --ldap help --- anonymize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 9d51a8979..f320e3e72 100755 --- a/anonymize.py +++ b/anonymize.py @@ -500,7 +500,8 @@ def add_options(self): r'this as user/host\@realm to user/, which would have exposed \'user\'' + \ '. Auto enables --email, --domain and --fqdn') self.add_opt('-L', '--ldap', action='store_true', - help='Apply LDAP anonymization (~100 attribs eg. CN, DN, OU, UID, sAMAccountName, memberOf...)') + help='Apply LDAP anonymization ' + \ + '(~100 attribs eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...)') self.add_opt('-E', '--email', action='store_true', help='Apply email format anonymization') #self.add_opt('-x', '--proxy', action='store_true', From b6fefcea8b9d594ec7376603bad9271d31776325 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Sep 2018 19:57:50 +0100 Subject: [PATCH 0006/2779] updated tests/test_anonymize.sh --- tests/test_anonymize.sh | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index ef1d02e69..1b41850c8 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -379,13 +379,28 @@ dest[95]="/usr/hdp/2.6.2.0-123/blah" src[96]="log4j-1.2.3.4.jar" dest[96]="log4j-1.2.3.4.jar" - # check escape codes get stripped if present (eg. if piping from grep --color-yes) #src[88]="some^[[01;31m^[[Khost^[[m^[[Kname:443" #src[88]="some\e[01;31m\e[Khost\e[m\e[K:443" src[99]="$(echo somehost:443 | grep --color=yes host)" dest[99]=":443" +src[100]="-Dhost=blah" +dest[100]="-Dhost=" + +src[101]="-Ddomain.com=blah" +dest[101]="-Ddomain.com=blah" + +src[102]="-Dhost.domain.com=blah" +dest[102]="-Dhost.domain.com=blah" + +# TODO: move proxy hosts to host matches and re-enable +#src[103]="proxy blah port 8080" +#dest[103]="proxy port " + +#src[104]="Connected to blah (1.2.3.4) port 8080" +#dest[104]="Connected to () port " + args="-aPe" test_anonymize(){ run++ @@ -447,27 +462,27 @@ echo echo "Running Tests preseving text without --network enabled:" echo # check normal don't strip these -src[101]="reading password from foo" -dest[101]="reading password from foo" +src[901]="reading password from foo" +dest[901]="reading password from foo" -src[102]="some description = blah, module = foo" -dest[102]="some description = blah, module = foo" +src[902]="some description = blah, module = foo" +dest[902]="some description = blah, module = foo" -args="-HKEiux" -run_tests 101 102 # ignore_run_unqualified +args="-HKEiu" +run_tests 901 902 # ignore_run_unqualified echo echo "Running Network Specific Tests:" echo # now check --network / --cisco / --juniper do strip these -src[103]="reading password from bar" -dest[103]="reading password " +src[903]="reading password from bar" +dest[903]="reading password " -src[104]="some description = blah, module=bar" -dest[104]="some description " +src[904]="some description = blah, module=bar" +dest[904]="some description " args="--network" -run_tests 103 104 # ignore_run_unqualified +run_tests 903 904 # ignore_run_unqualified if [ -n "$parallel" ]; then # can't trust exit code for parallel yet, only for quick local testing From fb93fb017e492f0d50d7f5db6517aa688ec2f230 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Sep 2018 19:59:00 +0100 Subject: [PATCH 0007/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 122b68e97..34304c0d0 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Environment variables are supported for convenience and also to hide credentials - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - ```--hash-hostnames``` - generates salted hashes that look like Docker temporary container ids so that vendors support teams can differentiate hosts in clusters + - ```--hash-hostnames``` - generates salted hostname hashes that look like Docker temporary container IDs so that vendors support teams can differentiate hosts in clusters - [Spark](https://spark.apache.org/) & Data Format Converters: - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter From 0d36a17bead518a360ddbcea4a6e130523d125c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Sep 2018 12:38:57 +0100 Subject: [PATCH 0008/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 34304c0d0..fff78deb2 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,8 @@ Environment variables are supported for convenience and also to hide credentials - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - ```--hash-hostnames``` - generates salted hostname hashes that look like Docker temporary container IDs so that vendors support teams can differentiate hosts in clusters + - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters +- ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - [Spark](https://spark.apache.org/) & Data Format Converters: - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter @@ -81,7 +82,6 @@ Environment variables are supported for convenience and also to hide credentials - ```xml_to_json.py``` - XML to JSON converter - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - see also ```validate_*.py``` further down for all these formats and more -- ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - [Ambari](https://hortonworks.com/apache/ambari/): - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API - list blueprints From e0ab8bb5ca0358615d81a9f04d90a8cf5a3f38b9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Sep 2018 12:43:02 +0100 Subject: [PATCH 0009/2779] updated README.md --- README.md | 178 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 91 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index fff78deb2..89db7c647 100644 --- a/README.md +++ b/README.md @@ -58,98 +58,102 @@ Environment variables are supported for convenience and also to hide credentials ### DevOps Python Tools ### -- ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) - - anonymizes: - - hostnames / domains / FQDNs - - email addresses - - IP + MAC addresses - - Kerberos principals - - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` - - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters -- ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename -- [Spark](https://spark.apache.org/) & Data Format Converters: - - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter - - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas - - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```json_to_xml.py``` - JSON to XML converter - - ```xml_to_json.py``` - XML to JSON converter - - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - - see also ```validate_*.py``` further down for all these formats and more -- [Ambari](https://hortonworks.com/apache/ambari/): - - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API - - list blueprints - - fetch all blueprints or a specific blueprint to local json files - - blueprint an existing cluster - - create a new cluster using a blueprint - - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control - - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool - - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API - - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API -- [Hadoop](http://hadoop.apache.org/) HDFS: - - ```hadoop_hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - ```hadoop_hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files -- [HBase](https://hbase.apache.org/): - - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables - - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) -- [OpenTSDB](http://opentsdb.net/): - - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending -- [Pig](https://pig.apache.org/) - - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop -- ```ipython-notebook-pyspark.py``` - per-user authenticated IPython Notebook + PySpark integration to allow each user to auto-create their own password protected IPython Notebook running Spark +- Linux: + - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - anonymizes: + - hostnames / domains / FQDNs + - email addresses + - IP + MAC addresses + - Kerberos principals + - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) + - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings + - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` + - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) + - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications + - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters + - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) +- Hadoop Ecosystem: + - [Spark](https://spark.apache.org/) & Data Format Converters: + - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter + - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter + - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas + - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas + - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter + - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter + - ```json_to_xml.py``` - JSON to XML converter + - ```xml_to_json.py``` - XML to JSON converter + - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - see also ```validate_*.py``` further down for all these formats and more + - [Ambari](https://hortonworks.com/apache/ambari/): + - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API + - list blueprints + - fetch all blueprints or a specific blueprint to local json files + - blueprint an existing cluster + - create a new cluster using a blueprint + - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control + - optionally strips out the excessive and overly specific configs to create generic more reusable templates + - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool + - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts + - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API + - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API + - [Hadoop](http://hadoop.apache.org/) HDFS: + - ```hadoop_hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - ```hadoop_hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - [HBase](https://hbase.apache.org/): + - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables + - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions + - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix + - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. + - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. + - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending + - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting + - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting + - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - [OpenTSDB](http://opentsdb.net/): + - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting + - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - [Pig](https://pig.apache.org/) + - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop + - ```ipython-notebook-pyspark.py``` - per-user authenticated IPython Notebook + PySpark integration to allow each user to auto-create their own password protected IPython Notebook running Spark + - ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) + - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required + - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA + - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA + - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA + - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) + - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node + - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator + - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server + - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node + - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node + - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (official `docker search` is limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` -- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required - - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) -- ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red -- ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI +- Travis CI: + - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI +- Data Validation (useful in CI): - ```validate_*.py``` - validate files, directory trees and/or standard input streams - - supports the following file formats: - - Avro - - CSV - - INI / Java Properties (also detects duplicate sections and duplicate keys within sections) - - JSON (both normal and json-doc-per-line bulk / big data format as found in MongoDB and Hadoop json data files) - - LDAP LDIF - - Parquet - - XML - - YAML - - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) - - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies -- ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - supports the following file formats: + - Avro + - CSV + - INI / Java Properties (also detects duplicate sections and duplicate keys within sections) + - JSON (both normal and json-doc-per-line bulk / big data format as found in MongoDB and Hadoop json data files) + - LDAP LDIF + - Parquet + - XML + - YAML + - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) + - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies ### Detailed Build Instructions From 35ec37a9b05ec29570e8e7794c11d7ae2a90d642 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Sep 2018 12:43:47 +0100 Subject: [PATCH 0010/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89db7c647..78b201d9f 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Environment variables are supported for convenience and also to hide credentials - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) -- Hadoop Ecosystem: +- Hadoop & NoSQL: - [Spark](https://spark.apache.org/) & Data Format Converters: - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter From dbea5a08282d208897689d1a562a9eae5011e518 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Sep 2018 12:44:28 +0100 Subject: [PATCH 0011/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78b201d9f..051f7610a 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ Environment variables are supported for convenience and also to hide credentials - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI - Data Validation (useful in CI): -- ```validate_*.py``` - validate files, directory trees and/or standard input streams + - ```validate_*.py``` - validate files, directory trees and/or standard input streams - supports the following file formats: - Avro - CSV From 7bafe1e3160326d35793906a115a435e1e0355e6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Sep 2018 12:45:41 +0100 Subject: [PATCH 0012/2779] updated anonymize.py --- anonymize.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/anonymize.py b/anonymize.py index f320e3e72..12b299c8c 100755 --- a/anonymize.py +++ b/anonymize.py @@ -89,7 +89,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.5' +__version__ = '0.8.6' class Anonymize(CLI): @@ -309,12 +309,12 @@ def __init__(self): pass_word_phrase=pass_word_phrase, pw=password_quoted), 'ip': r'(? Date: Wed, 3 Oct 2018 12:46:23 +0100 Subject: [PATCH 0013/2779] improved nt domain regex --- anonymize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/anonymize.py b/anonymize.py index 12b299c8c..35766fb88 100755 --- a/anonymize.py +++ b/anonymize.py @@ -89,7 +89,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.6' +__version__ = '0.8.7' class Anonymize(CLI): @@ -294,7 +294,8 @@ def __init__(self): 'user': r'(-{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/home/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), - 'user4': r'(?/) exclude patterns '>/' where we have already matched and token replaced 'user6': r'(?/){user}@'.format(user=user_regex), From 91ffbcf307aa36f5ba5883705e39708c195ed741 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Oct 2018 12:47:25 +0100 Subject: [PATCH 0014/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 1b41850c8..df553dba3 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -379,11 +379,14 @@ dest[95]="/usr/hdp/2.6.2.0-123/blah" src[96]="log4j-1.2.3.4.jar" dest[96]="log4j-1.2.3.4.jar" -# check escape codes get stripped if present (eg. if piping from grep --color-yes) -#src[88]="some^[[01;31m^[[Khost^[[m^[[Kname:443" -#src[88]="some\e[01;31m\e[Khost\e[m\e[K:443" -src[99]="$(echo somehost:443 | grep --color=yes host)" -dest[99]=":443" +src[97]='MYDOMAIN\n2018-10-03 01:23:45' +dest[97]='MYDOMAIN\n2018-10-03 01:23:45' + +src[98]='domain789012345\blah' +dest[98]='\' + +src[99]='domain7890123456\blah' +dest[99]='domain7890123456\blah' src[100]="-Dhost=blah" dest[100]="-Dhost=" @@ -394,6 +397,12 @@ dest[101]="-Ddomain.com=blah" src[102]="-Dhost.domain.com=blah" dest[102]="-Dhost.domain.com=blah" +# check escape codes get stripped if present (eg. if piping from grep --color-yes) +#src[88]="some^[[01;31m^[[Khost^[[m^[[Kname:443" +#src[88]="some\e[01;31m\e[Khost\e[m\e[K:443" +src[103]="$(echo somehost:443 | grep --color=yes host)" +dest[103]=":443" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From d141d1a4b9c2181ace4e4a580deba084b331a347 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Oct 2018 12:59:13 +0100 Subject: [PATCH 0015/2779] updated ambari_trigger_service_checks.py --- ambari_trigger_service_checks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ambari_trigger_service_checks.py b/ambari_trigger_service_checks.py index b22006408..abe593da1 100755 --- a/ambari_trigger_service_checks.py +++ b/ambari_trigger_service_checks.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2' +__version__ = '0.2.1' class AmbariTriggerServiceChecks(CLI): @@ -287,8 +287,9 @@ def request_service_checks(self, services): request_schedule_id = _['resources'][0]['RequestSchedule']['id'] log.info('RequestSchedule %s submitted', request_schedule_id) href = _['resources'][0]['href'] - assert href == self.url_base.rstrip('/') + '/clusters/{0}/request_schedules/{1}'\ - .format(self.cluster, request_schedule_id) + if href != self.url_base.rstrip('/') + '/clusters/{0}/request_schedules/{1}'\ + .format(self.cluster, request_schedule_id): + raise ValueError('href does not match expected request_schedules URI!') if self.watch: self.watch_scheduled_request(request_schedule_id) except (KeyError, ValueError) as _: From 8caee5f5b1ffc733eb5dac1e4003f5d41ef5a843 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Oct 2018 13:00:48 +0100 Subject: [PATCH 0016/2779] replaced assert --- find_active_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/find_active_server.py b/find_active_server.py index fe94f98fd..f3e5ea651 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -137,7 +137,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.2' +__version__ = '0.8.3' class FindActiveServer(CLI): @@ -419,7 +419,8 @@ def check_http(self, host, port, url_path=''): log.info('%s - checking regex against content', url) # if this ends up not being processed properly and remains a string instead # of the expected compiled regex, then .search() will hang - assert not isStr(self.regex) + if not isStr(self.regex): + die('non-string supplied for regex!') if self.regex.search(req.content): log.info('%s - regex matched http output', url) else: From bee2376c8465a8edd5e6e15c7fd776d1aaa2db7c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Oct 2018 14:41:36 +0100 Subject: [PATCH 0017/2779] updated tests/test_anonymize.sh --- tests/test_anonymize.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index df553dba3..4d1f5528a 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -403,6 +403,12 @@ dest[102]="-Dhost.domain.com=blah" src[103]="$(echo somehost:443 | grep --color=yes host)" dest[103]=":443" +src[104]='..., "user": "blah", "group": "blah2", "host": "blah3", ...' +dest[104]='..., "user": "", "group": "", "host": "", ...' + +src[105]='...,"owner":"blah","hostname":"blah2",...' +dest[105]='...,"owner":"","hostname":"",...' + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 9eb72395fc81786f1158d773e7ccd06bc77a3119 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Oct 2018 14:43:15 +0100 Subject: [PATCH 0018/2779] added json matches for user/owner, group and hostname --- anonymize.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 35766fb88..f8d228b23 100755 --- a/anonymize.py +++ b/anonymize.py @@ -89,7 +89,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.7' +__version__ = '0.8.8' class Anonymize(CLI): @@ -286,11 +286,13 @@ def __init__(self): # no \bhost - want to match KrbHost 'hostname5': r'(-host(?:name)?{sep}|host(?:name)?\s*=\s*)({hostname})'\ .format(sep=arg_sep, hostname=hostname_regex), + 'hostname6': r'(["\']host(?:name)?["\']\s*:\s*["\']?){hostname}'.format(hostname=hostname_regex), # use more permissive hostname_regex to catch Kerberos @REALM etc 'domain2': '@{host}'.format(host=hostname_regex), 'group': r'(-{group_name}{sep})\S+'.format(group_name=group_name, sep=arg_sep), 'group2': r'({group_name}{sep}){user}'.format(group_name=group_name, sep=arg_sep, user=user_regex), 'group3': r'for\s+group\s+{group}'.format(group=user_regex), + 'group4': r'(["\']{group_name}["\']\s*:\s*["\']?){group}'.format(group_name=group_name, group=user_regex), 'user': r'(-{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/home/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), @@ -299,6 +301,7 @@ def __init__(self): 'user5': r'for\s+user\s+{user}'.format(user=user_regex), # (?/) exclude patterns '>/' where we have already matched and token replaced 'user6': r'(?/){user}@'.format(user=user_regex), + 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'.format(user_name=user_name, user=user_regex), 'password': r'(-?{pass_word_phrase}{sep}){pw}'\ .format(pass_word_phrase=pass_word_phrase, sep=arg_sep, @@ -377,6 +380,7 @@ def __init__(self): 'hostname3': r'\1', 'hostname4': r'\\\\', 'hostname5': r'\1', + 'hostname6': r'\1', 'domain2': '@', 'port': ':', 'user': r'\1', @@ -385,9 +389,11 @@ def __init__(self): 'user4': r'\\', 'user5': 'for user ', 'user6': '@', + 'user7': r'\1', 'group': r'\1', 'group2': r'\1', 'group3': r'for group ', + 'group4': r'\1', 'password': r'\1', 'password2': r'\1:', 'password3': r'\1\2', From 0c07571743d600af46651851c2829211268508e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Oct 2018 15:40:20 +0100 Subject: [PATCH 0019/2779] updated tests/test_anonymize.sh --- tests/test_anonymize.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 4d1f5528a..8b731b6e9 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -409,6 +409,15 @@ dest[104]='..., "user": "", "group": "", "host": "", ...' src[105]='...,"owner":"blah","hostname":"blah2",...' dest[105]='...,"owner":"","hostname":"",...' +src[106]="ambari-sudo.sh" +dest[106]="ambari-sudo.sh" + +src[107]="hdfs://user/blah" +dest[107]="hdfs:///" + +src[108]="hdfs:///user/blah" +dest[108]="hdfs:///user/" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 2270dc6dbbb8358d8dd43af6d51ee261ca2be397 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Oct 2018 15:40:50 +0100 Subject: [PATCH 0020/2779] improved host lookbehinds to be simple common file extensions, added /user/ matching --- anonymize.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/anonymize.py b/anonymize.py index f8d228b23..cb9f31c9a 100755 --- a/anonymize.py +++ b/anonymize.py @@ -89,7 +89,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.8' +__version__ = '0.8.9' class Anonymize(CLI): @@ -137,8 +137,9 @@ def __init__(self): 'java_exceptions': False, 'python_tracebacks': False, } - self.negative_host_lookbehind = r'(?', 'port': ':', 'user': r'\1', - 'user2': '/home/', + 'user2': r'/\1/', 'user3': r'\1', 'user4': r'\\', 'user5': 'for user ', From d73483c4f76ae30a847feb8a6d492d1e368f95ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Oct 2018 11:11:06 +0100 Subject: [PATCH 0021/2779] updated test_validate_ini.sh --- tests/test_validate_ini.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh index 745b4caf1..32a658e9f 100755 --- a/tests/test_validate_ini.sh +++ b/tests/test_validate_ini.sh @@ -49,6 +49,10 @@ run_fail2(){ run_fail "${@/validate_ini/validate_ini2}" # ignore_run_unqualified } +if [ -f /etc/sssd/sssd.conf -a -r /etc/sssd/sssd.conf ]; then + run ./validate_ini.py /etc/sssd/sssd.conf +fi + run ./validate_ini.py --exclude "$exclude" . run_fail 2 ./validate_ini2.py --exclude "$exclude" . echo From 912a2d2bfbc671ac026b546b319b2c7ed8c41106 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 9 Oct 2018 12:15:59 +0100 Subject: [PATCH 0022/2779] added python 2 unidecode support to avoid regex module failures --- anonymize.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/anonymize.py b/anonymize.py index cb9f31c9a..10f040f77 100755 --- a/anonymize.py +++ b/anonymize.py @@ -61,6 +61,7 @@ from harisekhon.utils import \ isJavaException, \ isPythonTraceback, \ + isPythonMinVersion, \ isRegex, \ isStr, \ log, \ @@ -89,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.9' +__version__ = '0.9.0' class Anonymize(CLI): @@ -302,7 +303,8 @@ def __init__(self): 'user5': r'for\s+user\s+{user}'.format(user=user_regex), # (?/) exclude patterns '>/' where we have already matched and token replaced 'user6': r'(?/){user}@'.format(user=user_regex), - 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'.format(user_name=user_name, user=user_regex), + 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'\ + .format(user_name=user_name, user=user_regex), 'password': r'(-?{pass_word_phrase}{sep}){pw}'\ .format(pass_word_phrase=pass_word_phrase, sep=arg_sep, @@ -765,8 +767,10 @@ def anonymize(self, line): line_ending = match.group(1) if self.strip_cr: line_ending = '\n' - line = self.re_line_ending.sub('', line) + if not isPythonMinVersion(3): + line = line.decode('utf-8').encode('ascii', errors='?') line = strip_ansi_escape_codes(line) + line = self.re_line_ending.sub('', line) for _ in self.anonymizations: if not self.anonymizations[_]: continue From f4ede96d4ba0549410f1964bb956b5146fb1798d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 9 Oct 2018 13:44:37 +0100 Subject: [PATCH 0023/2779] fixed unicode error replace --- anonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonymize.py b/anonymize.py index 10f040f77..1d698a8ee 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.0' +__version__ = '0.9.1' class Anonymize(CLI): @@ -768,7 +768,7 @@ def anonymize(self, line): if self.strip_cr: line_ending = '\n' if not isPythonMinVersion(3): - line = line.decode('utf-8').encode('ascii', errors='?') + line = line.decode('utf-8').encode('ascii', errors='replace') line = strip_ansi_escape_codes(line) line = self.re_line_ending.sub('', line) for _ in self.anonymizations: From fa39c26393dc267fa8c618919dcf1b5a1ac044cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 14 Oct 2018 19:21:54 +0100 Subject: [PATCH 0024/2779] updated test_apache-drill.sh --- tests/test_apache-drill.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_apache-drill.sh b/tests/test_apache-drill.sh index b3192f68c..e08b42239 100755 --- a/tests/test_apache-drill.sh +++ b/tests/test_apache-drill.sh @@ -24,7 +24,7 @@ cd "$srcdir/.." section "A p a c h e D r i l l" -export APACHE_DRILL_VERSIONS="${@:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 latest}}" +export APACHE_DRILL_VERSIONS="${@:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 1.14 latest}}" APACHE_DRILL_HOST="${DOCKER_HOST:-${APACHE_DRILL_HOST:-${HOST:-localhost}}}" APACHE_DRILL_HOST="${APACHE_DRILL_HOST##*/}" From 0c3e2d8d5ba2a7f76532e5d92a9a55c969a41e77 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 15 Oct 2018 14:27:39 +0100 Subject: [PATCH 0025/2779] updated strip_ansi_escape_codes.py --- strip_ansi_escape_codes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/strip_ansi_escape_codes.py b/strip_ansi_escape_codes.py index b828faaa8..6fc517dc6 100755 --- a/strip_ansi_escape_codes.py +++ b/strip_ansi_escape_codes.py @@ -42,7 +42,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2' # pylint: disable=too-few-public-methods @@ -72,11 +72,11 @@ def run(self): die("path '%s' could not be determined as either a file or directory" % arg) for filename in self.args: if filename == '-': - for line in sys.stdin.readlines(): + for line in sys.stdin: print(strip_ansi_escape_codes(line), end='') else: - with open(filename) as _: - for line in _.readlines(): + with open(filename) as filehandle: + for line in filehandle: print(strip_ansi_escape_codes(line), end='') From 3a52c066c83fb31107eb07963df0ae6b1d0dff0c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 15 Oct 2018 14:27:45 +0100 Subject: [PATCH 0026/2779] updated tests/test_strip_ansi_escape_codes.sh --- tests/test_strip_ansi_escape_codes.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_strip_ansi_escape_codes.sh b/tests/test_strip_ansi_escape_codes.sh index ef088963c..b885c1878 100755 --- a/tests/test_strip_ansi_escape_codes.sh +++ b/tests/test_strip_ansi_escape_codes.sh @@ -39,8 +39,30 @@ if echo "some highlighted content" | echo "ANSI escape code stripping FAILED" exit 1 fi +hr + +tmp=$(mktemp /tmp/strip_ansi_escape_codes.XXXXX) +trap "rm $tmp" $TRAP_SIGNALS + +echo +echo "creating tmp file:" +echo "some highlighted content" | grep --color=yes highlighted > "$tmp" +hr + +echo +echo "checking stripping from file" +run++ +if ./strip_ansi_escape_codes.py "$tmp" | +tee /dev/stderr | + grep -q '^some highlighted content$'; then + echo "ANSI escape code stripping SUCCEEDED" + else + echo "ANSI escape code stripping FAILED" + exit 1 +fi echo echo "Total Tests run: $run_count" time_taken "$start_time" "All version tests for $name completed in" echo +untrap From 02fec8165a1d6c3b6ecd36428139c1bb182cd9ef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Dec 2018 18:23:21 +0000 Subject: [PATCH 0027/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index 0ccd31ac3..83252f774 100755 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -17,6 +17,7 @@ cyrus-sasl-devel gcc gcc-c++ git +krb5-devel python-devel unzip wget From 490eb2e1864ca38b65c6c8293025ff6867a49b6b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Dec 2018 18:24:26 +0000 Subject: [PATCH 0028/2779] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index 7f98d1c39..83618d4cd 100755 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -14,6 +14,7 @@ # ============================================================================ # build-essential +libkrb5-dev libsasl2-dev python-dev unzip From 02ed59d2206d77826b2e9a56f92ef268dabcd557 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Dec 2018 18:26:26 +0000 Subject: [PATCH 0029/2779] updated requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9ded348a5..9b1b84dec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ py>=1.4.31 #pyhs2>=0.6.0 #pytest>=3.0.0 #python-jenkins>=0.4.13 -#python-krbV>=1.0.90 +python-krbV>=1.0.90 # needed by avro python-snappy>=0.5 PyYAML>=3.11 @@ -40,7 +40,7 @@ requests>=2.11.1 sasl>=0.2.1 sh>=1.12.14 six>=1.10.0 -#snakebite[kerberos]>=2.11.0 +snakebite[kerberos]>=2.11.0 thrift-sasl>=0.2.1 thrift>=0.9.3 traceback2>=1.4.0 From 2e8cd5d7b6097a8438296d4d71690e5e619991f2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Dec 2018 18:37:10 +0000 Subject: [PATCH 0030/2779] updated tests/docker/hadoop-docker-compose.yml --- tests/docker/hadoop-docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/docker/hadoop-docker-compose.yml b/tests/docker/hadoop-docker-compose.yml index 63e9f53de..bef1fe74b 100644 --- a/tests/docker/hadoop-docker-compose.yml +++ b/tests/docker/hadoop-docker-compose.yml @@ -18,7 +18,7 @@ services: extends: file: common.yml service: common - image: harisekhon/hadoop:${VERSION:-latest} + image: harisekhon/hadoop-dev:${VERSION:-latest} ports: #- 8020 - 8042 @@ -31,6 +31,8 @@ services: - 50070 - 50075 - 50090 + volumes: + - ../..:/py networks: - hadoop From c58da919ac6a14108db1655e11ebc64cefef5635 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Dec 2018 20:12:09 +0000 Subject: [PATCH 0031/2779] added hdfs_find_replication_factor_1.py --- hdfs_find_replication_factor_1.py | 143 ++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100755 hdfs_find_replication_factor_1.py diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py new file mode 100755 index 000000000..5dec3c8ff --- /dev/null +++ b/hdfs_find_replication_factor_1.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2018-11-28 16:37:00 +0000 (Wed, 28 Nov 2018) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to find HDFS file with replication factor 1 + +These cause problems because taking a single datanode offline may result in alerts for files with missing blocks + +Uses any arguments are directory tree paths to starting scanning down. If no argument paths are given, searches under +top level directory / + +Uses Hadoop configuration files it expects to find in $HADOOP_HOME/conf to auto-detect +NameNodes HA, Kerberos etc (just kinit first) + +Optionally resets such files back to replication factor 3 if specifying --set-replication-factor-3 + +Tested on Hadoop 2.7 on HDP 2.6 with Kerberos + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +import time +import traceback +import krbV +import snakebite +from snakebite.client import AutoConfigClient +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, log_option, validate_int + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class HdfsFindReplicationFactor1(CLI): + + def __init__(self): + # Python 2.x + super(HdfsFindReplicationFactor1, self).__init__() + # Python 3.x + # super().__init__() + self.path_list = None + self.replication_factor = None + + def add_options(self): + super(HdfsFindReplicationFactor1, self).add_options() + self.add_opt('--hadoop-home', + help='Sets $HADOOP_HOME, expects to find config in $HADOOP_HOME/conf, ' + \ + 'otherwise inherits from environment or tries default paths') + #self.add_opt('--set-replication', metavar='N', type=int, + # help='Resets any files with replication factor 1 back to this replication factor (optional)') + + def process_options(self): + super(HdfsFindReplicationFactor1, self).process_options() + self.path_list = self.args + if not self.path_list: + self.path_list = ['/'] + self.replication_factor = self.get_opt('set_replication') + if self.replication_factor is not None: + validate_int(self.replication_factor, 'set replication', 2, 5) + hadoop_home = self.get_opt('hadoop_home') + if hadoop_home is not None: + os.environ['HADOOP_HOME'] = hadoop_home + hadoop_home_env = os.getenv('HADOOP_HOME') + log_option('HADOOP_HOME', hadoop_home_env) + if hadoop_home_env: + log.info('will search for Hadoop config in %s/conf', hadoop_home_env) + + def run(self): + log.info('initiating snakebite hdfs client') + try: + client = AutoConfigClient() + except krbV.Krb5Error as _: # pylint: disable=no-member + if self.verbose: + print('', file=sys.stderr) + print(_, file=sys.stderr) + start_time = time.time() + dir_count = 0 + file_count = 0 + repl1_count = 0 + for path in self.path_list: + try: + result_list = client.ls([path], recurse=True, include_toplevel=True, include_children=True) + for result in result_list: + if self.verbose: + print('.', file=sys.stderr, end='') + if result['block_replication'] == 0: + dir_count += 1 + continue + file_count += 1 + if result['block_replication'] == 1: + file_path = result['path'] + repl1_count += 1 + if self.verbose: + print('', file=sys.stderr) + print(file_path) + if self.replication_factor: + log.info('setting replication factor to {} on {}'\ + .format(self.replication_factor, file_path)) + # XXX: doesn't work + client.setrep([file_path], self.replication_factor, recurse=False) + except (snakebite.errors.FileNotFoundException, snakebite.errors.RequestError) as _: + if self.verbose: + print('', file=sys.stderr) + print(_, file=sys.stderr) + if self.verbose: + print('', file=sys.stderr) + secs = int(time.time() - start_time) + print('\nCompleted in {} secs\n'.format(secs), file=sys.stderr) + print('{} files with replication factor 1 out of {} files in {} dirs'\ + .format(repl1_count, file_count, dir_count), file=sys.stderr) + + +if __name__ == '__main__': + HdfsFindReplicationFactor1().main() From 8a0a34118749acef5769a3c90b09ab91280b71cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Dec 2018 20:13:00 +0000 Subject: [PATCH 0032/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6aa8b04c3..078956028 100755 --- a/Makefile +++ b/Makefile @@ -279,7 +279,7 @@ updatem: update-submodules clean: cd pylib && $(MAKE) clean @# the xargs option to ignore blank input doesn't work on Mac - @find . -maxdepth 3 -iname '*.py[co]' -o -iname '*.jy[co]' | xargs rm -f + @find . -iname '*.py[co]' -o -iname '*.jy[co]' | xargs rm -f @find . -type d -ipath '*/tests/*' -iname 'test-*spark*.avro' | xargs rm -rf @find . -type d -ipath '*/tests/*' -iname 'test-*spark*.parquet' | xargs rm -rf @rm -f parquet-tools-$(PARQUET_VERSION)-bin.zip From d64e3ce3bec06099283cb991063854ed6a43952a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Dec 2018 20:20:01 +0000 Subject: [PATCH 0033/2779] enabled --set-rep --- hdfs_find_replication_factor_1.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py index 5dec3c8ff..f6c290551 100755 --- a/hdfs_find_replication_factor_1.py +++ b/hdfs_find_replication_factor_1.py @@ -57,7 +57,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2' class HdfsFindReplicationFactor1(CLI): @@ -75,8 +75,8 @@ def add_options(self): self.add_opt('--hadoop-home', help='Sets $HADOOP_HOME, expects to find config in $HADOOP_HOME/conf, ' + \ 'otherwise inherits from environment or tries default paths') - #self.add_opt('--set-replication', metavar='N', type=int, - # help='Resets any files with replication factor 1 back to this replication factor (optional)') + self.add_opt('--set-replication', metavar='N', type=int, + help='Resets any files with replication factor 1 back to this replication factor (optional)') def process_options(self): super(HdfsFindReplicationFactor1, self).process_options() @@ -125,8 +125,8 @@ def run(self): if self.replication_factor: log.info('setting replication factor to {} on {}'\ .format(self.replication_factor, file_path)) - # XXX: doesn't work - client.setrep([file_path], self.replication_factor, recurse=False) + # returns a generator so cast to list to force evaluation + list(client.setrep([file_path], self.replication_factor, recurse=False)) except (snakebite.errors.FileNotFoundException, snakebite.errors.RequestError) as _: if self.verbose: print('', file=sys.stderr) From f7316d3be2b5a2b12724a18b4ce133e535e54e8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Dec 2018 20:37:58 +0000 Subject: [PATCH 0034/2779] updated find_active_server.py --- find_active_server.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/find_active_server.py b/find_active_server.py index f3e5ea651..94af0f75d 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -137,7 +137,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.3' +__version__ = '0.8.4' class FindActiveServer(CLI): @@ -419,8 +419,11 @@ def check_http(self, host, port, url_path=''): log.info('%s - checking regex against content', url) # if this ends up not being processed properly and remains a string instead # of the expected compiled regex, then .search() will hang - if not isStr(self.regex): - die('non-string supplied for regex!') + # now a precompiled regex + #if not isStr(self.regex): + # die('non-string supplied for regex!') + #if type(self.regex) != '_sre.SRE_Pattern': + # code_error('non-precompiled regex found!') if self.regex.search(req.content): log.info('%s - regex matched http output', url) else: From ef7b0446b1289b3523a8531079f575a3b288a534 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Dec 2018 18:55:26 +0000 Subject: [PATCH 0035/2779] updated hdfs_find_replication_factor_1.py --- hdfs_find_replication_factor_1.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py index f6c290551..516122e7b 100755 --- a/hdfs_find_replication_factor_1.py +++ b/hdfs_find_replication_factor_1.py @@ -57,7 +57,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2' +__version__ = '0.3' class HdfsFindReplicationFactor1(CLI): @@ -110,7 +110,7 @@ def run(self): try: result_list = client.ls([path], recurse=True, include_toplevel=True, include_children=True) for result in result_list: - if self.verbose: + if self.verbose and (dir_count + file_count) % 100 == 0: print('.', file=sys.stderr, end='') if result['block_replication'] == 0: dir_count += 1 @@ -125,8 +125,15 @@ def run(self): if self.replication_factor: log.info('setting replication factor to {} on {}'\ .format(self.replication_factor, file_path)) - # returns a generator so cast to list to force evaluation - list(client.setrep([file_path], self.replication_factor, recurse=False)) + # returns a generator so must evaluate in order to actually execute + # otherwise you find there is no effect on the replication factor + for _ in client.setrep([file_path], self.replication_factor, recurse=False): + if 'result' not in _: + print('WARNING: result field not found in setrep result: {}'.format(_), + file=sys.stderr) + continue + if not _['result']: + print('WARNING: failed to setrep: {}'.format(_)) except (snakebite.errors.FileNotFoundException, snakebite.errors.RequestError) as _: if self.verbose: print('', file=sys.stderr) From d2fdf5453eb136da4b650822820753c7c3d65e54 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 14 Jan 2019 21:08:59 +0000 Subject: [PATCH 0036/2779] fixed regex search --- find_active_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/find_active_server.py b/find_active_server.py index f3e5ea651..70f264486 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -137,7 +137,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.3' +__version__ = '0.8.4' class FindActiveServer(CLI): @@ -419,8 +419,8 @@ def check_http(self, host, port, url_path=''): log.info('%s - checking regex against content', url) # if this ends up not being processed properly and remains a string instead # of the expected compiled regex, then .search() will hang - if not isStr(self.regex): - die('non-string supplied for regex!') + if isStr(self.regex): + die('string found instead of expected compiled regex!') if self.regex.search(req.content): log.info('%s - regex matched http output', url) else: From b4c5300b8c8f4530f521f76ffef86df5d852b86f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 14 Jan 2019 21:09:37 +0000 Subject: [PATCH 0037/2779] updated for Hadoop 3.0 --- find_active_hadoop_namenode.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/find_active_hadoop_namenode.py b/find_active_hadoop_namenode.py index 6e6c668ad..ff578d9ef 100755 --- a/find_active_hadoop_namenode.py +++ b/find_active_hadoop_namenode.py @@ -32,7 +32,9 @@ Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print NO_AVAILABLE_SERVER. -Tested on Hadoop 2.7.3 on HDP 2.6.1 and Apache Hadoop 2.3, 2.4, 2.5, 2.6, 2.7, 2.8 +Tested on Hadoop 3.0, 3.1 + +For older Hadoop versions see adjacent find_active_hadoop2_namenode.py """ @@ -56,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.2' +__version__ = '0.8.0' class FindActiveHadoopNamenode(FindActiveServer): @@ -66,7 +68,7 @@ def __init__(self): super(FindActiveHadoopNamenode, self).__init__() # Python 3.x # super().__init__() - self.default_port = 50070 + self.default_port = 9870 self.protocol = 'http' self.url_path = '/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus' self.regex = r'"State"\s*:\s*"active"' From d756c6c9dd2ae3d21b61d61f25038d67fd6383c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 14 Jan 2019 21:09:49 +0000 Subject: [PATCH 0038/2779] added find_active_hadoop2_namenode.py --- find_active_hadoop2_namenode.py | 73 +++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 find_active_hadoop2_namenode.py diff --git a/find_active_hadoop2_namenode.py b/find_active_hadoop2_namenode.py new file mode 100755 index 000000000..bdb7ee5e4 --- /dev/null +++ b/find_active_hadoop2_namenode.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Tue Sep 5 10:49:49 CEST 2017 +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to return the active Hadoop Namenode from an argument list of hosts + +Can mix and match between a comma separated list of hosts (--host server1,server2 or contents of the $HOST +environment variable if not specified) and general free-form space separated arguments, which is useful if piping +a host list through xargs. + +Multi-threaded for speed and exits upon first available host response to minimize delay to ~ 1 second or less. + +Useful for simplying scripting or generically extending tools that predate Namenode High Availability + +By default checks the same --port on all servers. Hosts may have optional : suffixes added to individually +override each one. + +Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print +NO_AVAILABLE_SERVER. + +Tested on Hadoop 2.7.3 on HDP 2.6.1 and Apache Hadoop 2.3, 2.4, 2.5, 2.6, 2.7, 2.8 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +import traceback +#from random import shuffle +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from find_active_hadoop_namenode import FindActiveHadoopNamenode +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.7.2' + + +class FindActiveHadoop2Namenode(FindActiveHadoopNamenode): + + def __init__(self): + # Python 2.x + super(FindActiveHadoop2Namenode, self).__init__() + # Python 3.x + # super().__init__() + self.default_port = 50070 + + +if __name__ == '__main__': + FindActiveHadoop2Namenode().main() From 1d7a91b842ae93c316c763ba6cf36fca0d722a40 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 15 Jan 2019 20:27:20 +0000 Subject: [PATCH 0039/2779] updated find_active_server.py --- find_active_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/find_active_server.py b/find_active_server.py index 70f264486..edcf0fb49 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -333,7 +333,7 @@ def port_override(self, host): def finish(self, host, port=None): print(host, end='') if port is not None and port != self.port: - print(':{0}'.format(port)) + print(':{0}'.format(port), end='') print() sys.exit(0) From 0acb31dabaff70b401f8ceb690f6db7a43074f8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 15 Jan 2019 20:28:26 +0000 Subject: [PATCH 0040/2779] added find_active_cassandra.py --- find_active_cassandra.py | 83 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 find_active_cassandra.py diff --git a/find_active_cassandra.py b/find_active_cassandra.py new file mode 100755 index 000000000..67360ccc3 --- /dev/null +++ b/find_active_cassandra.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Wed Sep 13 13:58:21 CEST 2017 +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to return the first active Cassandra node from an argument list of hosts + +Can mix and match between a comma separated list of hosts (--host server1,server2 or contents of the $HOST +environment variable if not specified) and general free-form space separated arguments, which is useful if piping +a host list through xargs. + +Multi-threaded for speed and exits upon first available host response to minimize delay to ~ 1 second or less. + +Useful for simplying scripting or generically extending tools that don't support HBase High Availability directly + +By default checks the same --port on all servers. Hosts may have optional : suffixes added to individually +override each one. + +Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print +NO_AVAILABLE_SERVER. + +Tested on Cassandra 3.11 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +import traceback +#from random import shuffle +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from find_active_server import FindActiveServer +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.7.1' + + +class FindActiveCassandra(FindActiveServer): + + def __init__(self): + # Python 2.x + super(FindActiveCassandra, self).__init__() + # Python 3.x + # super().__init__() + # TCP only check - JMX port, could just as easily use CQL 9042 + self.default_port = 7199 + self.protocol = None + + def add_options(self): + self.add_hostoption(name='Cassandra', default_port=self.default_port) + #self.add_ssl_opt() + self.add_common_opts() + + def process_options(self): + self.validate_common_opts() + + +if __name__ == '__main__': + FindActiveCassandra().main() From 39888d13184d3d87ff74a6b52aaa85035f7d04a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 15 Jan 2019 20:43:57 +0000 Subject: [PATCH 0041/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 051f7610a..54de3173f 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,7 @@ Environment variables are supported for convenience and also to hide credentials - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node + - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server From 0061e8db5419ff16cafefa691e9ee6ec48d1991e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Jan 2019 20:37:13 +0000 Subject: [PATCH 0042/2779] updated tests/test_apache-drill.sh --- tests/test_apache-drill.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_apache-drill.sh b/tests/test_apache-drill.sh index e08b42239..03c6539a8 100755 --- a/tests/test_apache-drill.sh +++ b/tests/test_apache-drill.sh @@ -24,7 +24,7 @@ cd "$srcdir/.." section "A p a c h e D r i l l" -export APACHE_DRILL_VERSIONS="${@:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 1.14 latest}}" +export APACHE_DRILL_VERSIONS="${@:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 1.14 1.15 latest}}" APACHE_DRILL_HOST="${DOCKER_HOST:-${APACHE_DRILL_HOST:-${HOST:-localhost}}}" APACHE_DRILL_HOST="${APACHE_DRILL_HOST##*/}" From a95886f039bdb9dbfc32d9a2e6b8a7a7e3f351cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Jan 2019 20:37:31 +0000 Subject: [PATCH 0043/2779] updated find_active_apache_drill.py --- find_active_apache_drill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/find_active_apache_drill.py b/find_active_apache_drill.py index 92a07bddb..ff4461960 100755 --- a/find_active_apache_drill.py +++ b/find_active_apache_drill.py @@ -32,7 +32,7 @@ Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print NO_AVAILABLE_SERVER. -Tested on Apache Drill 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 1.13 +Tested on Apache Drill 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 1.13, 1.14, 1.15 """ From ef511abb8a206e1168ccff06d3f8c8cdb8e67c3f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Jan 2019 20:37:40 +0000 Subject: [PATCH 0044/2779] updated find_active_apache_drill2.py --- find_active_apache_drill2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/find_active_apache_drill2.py b/find_active_apache_drill2.py index aaef3ad2a..4dcdf8219 100755 --- a/find_active_apache_drill2.py +++ b/find_active_apache_drill2.py @@ -32,7 +32,7 @@ Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print NO_AVAILABLE_SERVER. -Tested on Apache Drill 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 1.13 +Tested on Apache Drill 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 1.13, 1.14, 1.15 (API endpoint is not available in versions <= 1.6, see find_active_apache_drill.py instead) From a6d6aae79967083d0a12e89924b051de5197fd3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Feb 2019 01:11:50 +0000 Subject: [PATCH 0045/2779] added crunch_accounting_csv_converter.py --- crunch_accounting_csv_converter.py | 208 +++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100755 crunch_accounting_csv_converter.py diff --git a/crunch_accounting_csv_converter.py b/crunch_accounting_csv_converter.py new file mode 100755 index 000000000..01a36f3d9 --- /dev/null +++ b/crunch_accounting_csv_converter.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-02-04 23:24:30 +0000 (Mon, 04 Feb 2019) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to convert Credit Card CSVs to Crunch Accounting standard format - extracts the important fields + +File arguments are read one by one and converted and output to a file of the same name with _crunch.csv at the end + +If there is no balance column, you will prompted to provide the initial balance from which to calculate the running +balance column for Crunch, In that case if specifying multiple CSV statements as arguments, they must be given in +chronological order in order for one statement's running balance to flow on to the next in the correct order + +Can be easily extended for other banks CSV formats, as it's just simple matching of the headers, all that is required +is a sample + +Tested on Barclaycard Commercial statement CSV exports + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import csv +import os +import re +import sys +import traceback +from decimal import Decimal +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, isChars + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class CrunchAccountingCsvConverter(CLI): + + def __init__(self): + # Python 2.x + super(CrunchAccountingCsvConverter, self).__init__() + # Python 3.x + # super().__init__() + self.verbose = 2 + self.running_balance = None + self.default_timeout = None + self.credit_card = False + + def add_options(self): + super(CrunchAccountingCsvConverter, self).add_options() + self.add_opt('-c', '--credit-card', action='store_true', + help='Credit Card statements (inverts transactions against running balance)') + + def process_options(self): + super(CrunchAccountingCsvConverter, self).process_options() + self.credit_card = self.get_opt('credit_card') + + def run(self): + if not self.args: + self.usage('no file arguments specified') + for filename in self.args: + target_filename = '{}_crunch.csv'.format(re.sub(r'\.csv$', '', filename)) + log.info("converting file '%s' => '%s'", filename, target_filename) + if not self.convert(filename, target_filename): + log.error("FAILED to convert filename '%s'", filename) + + def convert(self, filename, target_filename): + csvreader = self.get_csvreader(filename) + if not csvreader: + return False + count = 0 + (positions, balance_position) = self.detect_columns(csvreader) + csvwriter = csv.writer(open(target_filename, 'w')) + for row in csvreader: + count += 1 + if balance_position is not None: + balance = row[balance_position] + elif self.running_balance is not None: + self.update_balance(row[positions['amount']]) + balance = self.running_balance + else: + log.error('no balance column found and no running balance given') + sys.exit(2) + csvwriter.writerow( + [ + row[positions['date']], + row[positions['desc']], + row[positions['amount']], + balance + ] + ) + log.info('%s: %s CSV lines processed', os.path.basename(filename), count) + return True + + def update_balance(self, amount): + if self.credit_card: + self.running_balance -= Decimal(amount) + else: + self.running_balance += Decimal(amount) + + def detect_columns(self, csvreader): + headers = csvreader.next() + positions = {'date': None, 'desc': None, 'amount': None} + balance_position = None + for (position, value) in enumerate(headers): + if 'Date' in value: + positions['date'] = position + elif 'Merchant Name' in value: + positions['desc'] = position + elif 'Amount' in value: + positions['amount'] = position + elif 'Balance' in value: + balance_position = position + for pos in positions: + if positions[pos] is None: + log.error('field %s not found', pos) + return False + if balance_position is None: + print('Running balance column not found\nPlease give starting balance to auto-calculate it: ', end='') + self.running_balance = sys.stdin.readline().strip() + try: + self.running_balance = Decimal(self.running_balance) + except ValueError as _: + log.error('INVALID starting balance %s, must be in a decimal number: %s', self.running_balance, _) + sys.exit(1) + return (positions, balance_position) + + @staticmethod + def get_csvreader(filename): + try: + filehandle = open(filename) + except IOError as _: + log.error(_) + return None + filename = os.path.basename(filename) + try: + dialect = csv.Sniffer().sniff(filehandle.read(1024)) + # this will raise an Error if invalid + dialect.strict = True + filehandle.seek(0) + csvreader = csv.reader(filehandle, dialect) + except csv.Error as _: + log.warning('file %s: %s', filename, _) + return None + csvreader = CrunchAccountingCsvConverter.validate_csvreader(csvreader, filename) + filehandle.seek(0) + return csvreader + + @staticmethod + def validate_csvreader(csvreader, filename): + count = 0 + try: + # csvreader doesn't seem to generate any errors ever :-( + # csv module allows entire lines of json/xml/yaml to go in as a single field + # Adding some invalidations manually + for field_list in csvreader: + # list of fields with no separator information + log.debug("line: %s", field_list) + # make it fail if there is only a single field on any line + if len(field_list) < 2: + log.error("less than 2 fields detected, aborting conversion of file '%s'", filename) + return None + # it's letting JSON through :-/ + if field_list[0] == '{': + log.info('JSON opening braces detected, not a CSV?') + return None + # extra protection along the same lines as anti-json: + # the first char of field should be alphanumeric, not syntax + # however instead of isAlnum allow quotes for quoted CSVs to pass validation + if not isChars(field_list[0][0], 'A-Za-z0-9"'): + log.info('non-alphanumeric / quote opening character detected in CSV') + return None + count += 1 + except csv.Error as _: + log.warning('file %s, line %s: %s', filename, csvreader.line_num, _) + return None + if count == 0: + log.error('zero lines detected, blank input is not valid CSV') + return None + return csvreader + + +if __name__ == '__main__': + CrunchAccountingCsvConverter().main() From 638c7f38388a1a52b7b18686e3a565c5c7d8d82f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Feb 2019 01:49:59 +0000 Subject: [PATCH 0046/2779] added statement reversing support and credit card inversion of amounts --- crunch_accounting_csv_converter.py | 31 +++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/crunch_accounting_csv_converter.py b/crunch_accounting_csv_converter.py index 01a36f3d9..a5c80c716 100755 --- a/crunch_accounting_csv_converter.py +++ b/crunch_accounting_csv_converter.py @@ -17,7 +17,8 @@ """ -Tool to convert Credit Card CSVs to Crunch Accounting standard format - extracts the important fields +Tool to convert Credit Card CSVs with headers to Crunch Accounting standard format - extracts the important fields, +calculates the running balance if necessary File arguments are read one by one and converted and output to a file of the same name with _crunch.csv at the end @@ -29,6 +30,7 @@ is a sample Tested on Barclaycard Commercial statement CSV exports +(Barclaycard lists entries in reverse chronological order so need to use --credit and --reverse-order) """ @@ -41,6 +43,7 @@ import os import re import sys +import tempfile import traceback from decimal import Decimal srcdir = os.path.abspath(os.path.dirname(__file__)) @@ -55,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2' class CrunchAccountingCsvConverter(CLI): @@ -69,15 +72,19 @@ def __init__(self): self.running_balance = None self.default_timeout = None self.credit_card = False + self.reverse_order = False def add_options(self): super(CrunchAccountingCsvConverter, self).add_options() self.add_opt('-c', '--credit-card', action='store_true', help='Credit Card statements (inverts transactions against running balance)') + self.add_opt('-r', '--reverse-order', action='store_true', + help='Statement entries are in reverse chronological order (eg. Barclaycard)') def process_options(self): super(CrunchAccountingCsvConverter, self).process_options() self.credit_card = self.get_opt('credit_card') + self.reverse_order = self.get_opt('reverse_order') def run(self): if not self.args: @@ -89,12 +96,15 @@ def run(self): log.error("FAILED to convert filename '%s'", filename) def convert(self, filename, target_filename): + if self.reverse_order: + filename = self.reverse_contents(filename) csvreader = self.get_csvreader(filename) if not csvreader: return False count = 0 (positions, balance_position) = self.detect_columns(csvreader) csvwriter = csv.writer(open(target_filename, 'w')) + csvwriter.writerow(['Date', 'Description', 'Amount', 'Balance']) for row in csvreader: count += 1 if balance_position is not None: @@ -109,19 +119,34 @@ def convert(self, filename, target_filename): [ row[positions['date']], row[positions['desc']], - row[positions['amount']], + self.amount(row[positions['amount']]), balance ] ) log.info('%s: %s CSV lines processed', os.path.basename(filename), count) return True + def amount(self, amount): + if self.credit_card: + return -Decimal(amount) + return amount + def update_balance(self, amount): if self.credit_card: self.running_balance -= Decimal(amount) else: self.running_balance += Decimal(amount) + @staticmethod + def reverse_contents(filename): + lines = open(filename).readlines() + lines = [lines[0]] + list(reversed(lines[1:])) + (_, tmp_filename) = tempfile.mkstemp(text=True) + filehandle = open(tmp_filename, 'w') + for line in lines: + filehandle.write(line) + return tmp_filename + def detect_columns(self, csvreader): headers = csvreader.next() positions = {'date': None, 'desc': None, 'amount': None} From 15760e8b74f754dcd07d67a997745eb066c44707 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Feb 2019 02:09:29 +0000 Subject: [PATCH 0047/2779] added --starting-balance instead of prompting for it, and only checking it once to allow running through multiple CSVs --- crunch_accounting_csv_converter.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/crunch_accounting_csv_converter.py b/crunch_accounting_csv_converter.py index a5c80c716..b7d333f0c 100755 --- a/crunch_accounting_csv_converter.py +++ b/crunch_accounting_csv_converter.py @@ -22,9 +22,9 @@ File arguments are read one by one and converted and output to a file of the same name with _crunch.csv at the end -If there is no balance column, you will prompted to provide the initial balance from which to calculate the running -balance column for Crunch, In that case if specifying multiple CSV statements as arguments, they must be given in -chronological order in order for one statement's running balance to flow on to the next in the correct order +If there is no balance column you must provide a --starting-balance from which to calculate the running balance column +for Crunch, In that case if specifying multiple CSV statements as arguments, they must be given in chronological order +in order for one statement's running balance to flow on to the next in the correct order Can be easily extended for other banks CSV formats, as it's just simple matching of the headers, all that is required is a sample @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2' +__version__ = '0.3' class CrunchAccountingCsvConverter(CLI): @@ -80,11 +80,19 @@ def add_options(self): help='Credit Card statements (inverts transactions against running balance)') self.add_opt('-r', '--reverse-order', action='store_true', help='Statement entries are in reverse chronological order (eg. Barclaycard)') + self.add_opt('-s', '--starting-balance', help='Starting Balance (used if no balance column is found') def process_options(self): super(CrunchAccountingCsvConverter, self).process_options() self.credit_card = self.get_opt('credit_card') self.reverse_order = self.get_opt('reverse_order') + self.running_balance = self.get_opt('starting_balance') + if self.running_balance is not None: + try: + self.running_balance = Decimal(self.running_balance) + except ValueError as _: + log.error('INVALID starting balance %s, must be in a decimal number: %s', self.running_balance, _) + sys.exit(1) def run(self): if not self.args: @@ -164,14 +172,8 @@ def detect_columns(self, csvreader): if positions[pos] is None: log.error('field %s not found', pos) return False - if balance_position is None: - print('Running balance column not found\nPlease give starting balance to auto-calculate it: ', end='') - self.running_balance = sys.stdin.readline().strip() - try: - self.running_balance = Decimal(self.running_balance) - except ValueError as _: - log.error('INVALID starting balance %s, must be in a decimal number: %s', self.running_balance, _) - sys.exit(1) + if balance_position is None and self.running_balance is None: + self.usage('no balance column detected, please specify --starting-balance') return (positions, balance_position) @staticmethod From 57db98898c389313f0ecd3cff9c15358015031da Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Feb 2019 18:37:15 +0000 Subject: [PATCH 0048/2779] updated ambari_cancel_all_requests.sh --- ambari_cancel_all_requests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ambari_cancel_all_requests.sh b/ambari_cancel_all_requests.sh index 667f0c80e..dd031a35d 100755 --- a/ambari_cancel_all_requests.sh +++ b/ambari_cancel_all_requests.sh @@ -51,7 +51,7 @@ if [ $# -gt 5 ]; then fi echo "querying Ambari for request IDs" -curl -u "$AMBARI_USER:$AMBARI_PASSWORD" "http://$AMBARI_HOST:$AMBARI_PORT/api/v1/clusters/$AMBARI_CLUSTER/requests" | +curl -u "$AMBARI_USER:$AMBARI_PASSWORD" "$PROTOCOL://$AMBARI_HOST:$AMBARI_PORT/api/v1/clusters/$AMBARI_CLUSTER/requests" | grep id | awk '{print $3}' | while read id; do From 8d3980c11248431c7255f1075fca0625177f1557 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Feb 2019 18:38:07 +0000 Subject: [PATCH 0049/2779] updated ambari_cancel_all_requests.sh --- ambari_cancel_all_requests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ambari_cancel_all_requests.sh b/ambari_cancel_all_requests.sh index dd031a35d..460e2b76f 100755 --- a/ambari_cancel_all_requests.sh +++ b/ambari_cancel_all_requests.sh @@ -56,5 +56,6 @@ grep id | awk '{print $3}' | while read id; do echo "requesting cancellation of request $id" + # Unfortunately this call returns 200 OK even if your user account doesn't have permission to do this operation, but will fail on the requests query above for wrong username/password curl -u "$AMBARI_USER:$AMBARI_PASSWORD" -i -H "X-Requested-By: $AMBARI_USER ($USER)" -X PUT -d '{"Requests":{"request_status":"ABORTED","abort_reason":"Aborted by user"}}' "$PROTOCOL://$AMBARI_HOST:$AMBARI_PORT/api/v1/clusters/$AMBARI_CLUSTER/requests/$id" done From 35067f0e3b76acfbed4228cd92b9a20820742e3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Feb 2019 18:44:29 +0000 Subject: [PATCH 0050/2779] slightly simplified and renamed class --- crunch_accounting_csv_converter.py | 36 ++++++++++++++---------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/crunch_accounting_csv_converter.py b/crunch_accounting_csv_converter.py index b7d333f0c..1063b35b6 100755 --- a/crunch_accounting_csv_converter.py +++ b/crunch_accounting_csv_converter.py @@ -17,8 +17,10 @@ """ -Tool to convert Credit Card CSVs with headers to Crunch Accounting standard format - extracts the important fields, -calculates the running balance if necessary +Tool to convert Bank or Credit Card CSV Statements with headers to Crunch Accounting standard format +for importing to Crunch for accounts reconciliation + +Extracts the important fields, calculates the running balance if necessary File arguments are read one by one and converted and output to a file of the same name with _crunch.csv at the end @@ -26,8 +28,9 @@ for Crunch, In that case if specifying multiple CSV statements as arguments, they must be given in chronological order in order for one statement's running balance to flow on to the next in the correct order -Can be easily extended for other banks CSV formats, as it's just simple matching of the headers, all that is required -is a sample +Can easily be extended for other Banks CSV formats, as it's just simple matching of the headers + +For additional formats just raise an github issue request with a sample CSV Tested on Barclaycard Commercial statement CSV exports (Barclaycard lists entries in reverse chronological order so need to use --credit and --reverse-order) @@ -58,14 +61,14 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3' +__version__ = '0.4' -class CrunchAccountingCsvConverter(CLI): +class CrunchAccountingCsvStatementConverter(CLI): def __init__(self): # Python 2.x - super(CrunchAccountingCsvConverter, self).__init__() + super(CrunchAccountingCsvStatementConverter, self).__init__() # Python 3.x # super().__init__() self.verbose = 2 @@ -75,7 +78,7 @@ def __init__(self): self.reverse_order = False def add_options(self): - super(CrunchAccountingCsvConverter, self).add_options() + super(CrunchAccountingCsvStatementConverter, self).add_options() self.add_opt('-c', '--credit-card', action='store_true', help='Credit Card statements (inverts transactions against running balance)') self.add_opt('-r', '--reverse-order', action='store_true', @@ -83,7 +86,7 @@ def add_options(self): self.add_opt('-s', '--starting-balance', help='Starting Balance (used if no balance column is found') def process_options(self): - super(CrunchAccountingCsvConverter, self).process_options() + super(CrunchAccountingCsvStatementConverter, self).process_options() self.credit_card = self.get_opt('credit_card') self.reverse_order = self.get_opt('reverse_order') self.running_balance = self.get_opt('starting_balance') @@ -115,10 +118,11 @@ def convert(self, filename, target_filename): csvwriter.writerow(['Date', 'Description', 'Amount', 'Balance']) for row in csvreader: count += 1 + amount = self.amount(row[positions['amount']]) if balance_position is not None: balance = row[balance_position] elif self.running_balance is not None: - self.update_balance(row[positions['amount']]) + self.running_balance += amount balance = self.running_balance else: log.error('no balance column found and no running balance given') @@ -127,7 +131,7 @@ def convert(self, filename, target_filename): [ row[positions['date']], row[positions['desc']], - self.amount(row[positions['amount']]), + amount, balance ] ) @@ -139,12 +143,6 @@ def amount(self, amount): return -Decimal(amount) return amount - def update_balance(self, amount): - if self.credit_card: - self.running_balance -= Decimal(amount) - else: - self.running_balance += Decimal(amount) - @staticmethod def reverse_contents(filename): lines = open(filename).readlines() @@ -193,7 +191,7 @@ def get_csvreader(filename): except csv.Error as _: log.warning('file %s: %s', filename, _) return None - csvreader = CrunchAccountingCsvConverter.validate_csvreader(csvreader, filename) + csvreader = CrunchAccountingCsvStatementConverter.validate_csvreader(csvreader, filename) filehandle.seek(0) return csvreader @@ -232,4 +230,4 @@ def validate_csvreader(csvreader, filename): if __name__ == '__main__': - CrunchAccountingCsvConverter().main() + CrunchAccountingCsvStatementConverter().main() From 98ca773f868041bd685856e0030ed345bc6d48ad Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Feb 2019 18:44:36 +0000 Subject: [PATCH 0051/2779] renamed crunch_accounting_csv_converter.py to crunch_accounting_csv_statement_converter.py --- ...v_converter.py => crunch_accounting_csv_statement_converter.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crunch_accounting_csv_converter.py => crunch_accounting_csv_statement_converter.py (100%) diff --git a/crunch_accounting_csv_converter.py b/crunch_accounting_csv_statement_converter.py similarity index 100% rename from crunch_accounting_csv_converter.py rename to crunch_accounting_csv_statement_converter.py From 92de35168f3d4af53e0c92a5b7de073d46384152 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Feb 2019 18:45:32 +0000 Subject: [PATCH 0052/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 1063b35b6..f2630e4e2 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -135,7 +135,7 @@ def convert(self, filename, target_filename): balance ] ) - log.info('%s: %s CSV lines processed', os.path.basename(filename), count) + log.info('%s CSV lines processed', count) return True def amount(self, amount): From b61f5b49090239f12860614acbc88417b33964ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Feb 2019 18:46:56 +0000 Subject: [PATCH 0053/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index f2630e4e2..1d36991cc 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -103,7 +103,9 @@ def run(self): for filename in self.args: target_filename = '{}_crunch.csv'.format(re.sub(r'\.csv$', '', filename)) log.info("converting file '%s' => '%s'", filename, target_filename) - if not self.convert(filename, target_filename): + if self.convert(filename, target_filename): + log.info("converted '%s' => '%s'", filename, target_filename) + else: log.error("FAILED to convert filename '%s'", filename) def convert(self, filename, target_filename): From 3fad4a643d2f66b2971a26102ea85a4832e11abc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Feb 2019 12:16:26 +0000 Subject: [PATCH 0054/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 1d36991cc..0ddfe97c8 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4' +__version__ = '0.5' class CrunchAccountingCsvStatementConverter(CLI): @@ -157,6 +157,9 @@ def reverse_contents(filename): def detect_columns(self, csvreader): headers = csvreader.next() + if headers[0][0] == '{': + log.error('JSON opening braces detected, not a CSV?') + return False positions = {'date': None, 'desc': None, 'amount': None} balance_position = None for (position, value) in enumerate(headers): @@ -208,18 +211,14 @@ def validate_csvreader(csvreader, filename): # list of fields with no separator information log.debug("line: %s", field_list) # make it fail if there is only a single field on any line - if len(field_list) < 2: - log.error("less than 2 fields detected, aborting conversion of file '%s'", filename) - return None - # it's letting JSON through :-/ - if field_list[0] == '{': - log.info('JSON opening braces detected, not a CSV?') + if len(field_list) < 3: + log.error("less than 3 fields detected, aborting conversion of file '%s'", filename) return None # extra protection along the same lines as anti-json: # the first char of field should be alphanumeric, not syntax # however instead of isAlnum allow quotes for quoted CSVs to pass validation if not isChars(field_list[0][0], 'A-Za-z0-9"'): - log.info('non-alphanumeric / quote opening character detected in CSV') + log.error('non-alphanumeric / quote opening character detected in CSV') return None count += 1 except csv.Error as _: From b3654f1b815780ed72bc936a577a13c8410b1719 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Feb 2019 20:49:07 +0000 Subject: [PATCH 0055/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9b1b84dec..7af85c152 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,7 @@ py>=1.4.31 python-krbV>=1.0.90 # needed by avro python-snappy>=0.5 -PyYAML>=3.11 +PyYAML>=3.13 requests>=2.11.1 sasl>=0.2.1 sh>=1.12.14 From 6dce82a7fb703bfb4bcffedd2c191cc2d5585a0c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 10 Feb 2019 20:33:43 +0000 Subject: [PATCH 0056/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1c4746337..07cbd5c4d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1c4746337785c416fb535d16d5ec1f8455c96742 +Subproject commit 07cbd5c4dc12ba427ac71c09c5c35977cd47155a From 55ffb5f0bb25aefed6bddbba829548d40a09bdfc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 10 Feb 2019 20:51:05 +0000 Subject: [PATCH 0057/2779] updated Makefile --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 078956028..8c39799f1 100755 --- a/Makefile +++ b/Makefile @@ -126,8 +126,6 @@ python: # LD_RUN_PATH=/usr/local/include pip install snappy # $(SUDO_PIP) pip install --upgrade -r requirements.txt - # prevents https://urllib3.readthedocs.io/en/latest/security.html#insecureplatformwarning - $(SUDO_PIP) pip install --upgrade ndg-httpsclient # for ipython-notebook-pyspark.py #$(SUDO_PIP) pip install jinja2 # HiveServer2 From d24d9440c7ef4c666cc0a13d916f00fd10ef4684 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 12 Feb 2019 10:59:11 +0000 Subject: [PATCH 0058/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 0ddfe97c8..2b58c6b92 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -33,7 +33,7 @@ For additional formats just raise an github issue request with a sample CSV Tested on Barclaycard Commercial statement CSV exports -(Barclaycard lists entries in reverse chronological order so need to use --credit and --reverse-order) +(Barclaycard lists entries in reverse chronological order so you'll need to use --reverse-order, as well as --credit-card) """ From 9c50737adf55fce8e41e5aa614f9f05026018d74 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 13 Feb 2019 00:30:45 +0000 Subject: [PATCH 0059/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 2b58c6b92..bbbd98f6d 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -33,7 +33,7 @@ For additional formats just raise an github issue request with a sample CSV Tested on Barclaycard Commercial statement CSV exports -(Barclaycard lists entries in reverse chronological order so you'll need to use --reverse-order, as well as --credit-card) +(Barclaycard lists entries in reverse chronological order so specify --reverse-order as well as --credit-card) """ From d362843a8f953d32188f31612c57a2a8714e3437 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 13 Feb 2019 18:40:56 +0000 Subject: [PATCH 0060/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 07cbd5c4d..1bbfdc7bc 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 07cbd5c4dc12ba427ac71c09c5c35977cd47155a +Subproject commit 1bbfdc7bc6b6106ff674251aff47e64e206ce1eb From 266d0f4ddde08b998f56611608d1f2098249b478 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 13 Feb 2019 18:41:15 +0000 Subject: [PATCH 0061/2779] deduped pip modules with pylib --- requirements.txt | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7af85c152..992242e52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ avro>=1.8.1 bitarray>=0.8.1 #cassandra-driver>=3.6.0 dicttoxml>=1.7.4 -#docker-py>=1.9.0 # Elasticsearch library must match major version :( # for Elasticsearch 2.x #elasticsearch>=2.0.0,<3.0.0 @@ -20,30 +19,20 @@ ldif3>=3.2.2 linecache2>=1.0.0 MarkupSafe>=0.23 #Markdown>=2.6.8 -nose>=1.3.7 numpy>=1.8.0rc1 psutil>=4.3.0 -py>=1.4.31 # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 #python-ldap>=2.4.41 # PyLint fails some checks in Travis that pass locally -#pylint==1.5.2 #pyhs2>=0.6.0 -#pytest>=3.0.0 #python-jenkins>=0.4.13 python-krbV>=1.0.90 # needed by avro python-snappy>=0.5 -PyYAML>=3.13 -requests>=2.11.1 sasl>=0.2.1 sh>=1.12.14 -six>=1.10.0 snakebite[kerberos]>=2.11.0 thrift-sasl>=0.2.1 thrift>=0.9.3 -traceback2>=1.4.0 -unittest2>=1.1.0 -#wheel>=0.29.0 xmltodict>=0.10.2 From 0711387bc2d5a3877f1d81c526676a3fa675b221 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Feb 2019 00:41:19 +0000 Subject: [PATCH 0062/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a2300e7cc..e7b72d5dc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a2300e7cc4d4fdcd5f87ddba5da99d750ba9d3c5 +Subproject commit e7b72d5dc002ee32c7fbe21abed83568b23f881f From ff94da5b0730536f05edf36f04134250037b71e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Feb 2019 00:41:19 +0000 Subject: [PATCH 0063/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1bbfdc7bc..088904d4d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1bbfdc7bc6b6106ff674251aff47e64e206ce1eb +Subproject commit 088904d4d6049f953494b7921f1f011b30bb1a0f From b565f89ce30ac64e991da436ae0ea6991aed86e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Feb 2019 01:05:43 +0000 Subject: [PATCH 0064/2779] added setup/rpm-packages-pip.txt --- setup/rpm-packages-pip.txt | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 setup/rpm-packages-pip.txt diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt new file mode 100755 index 000000000..7e62dfc4d --- /dev/null +++ b/setup/rpm-packages-pip.txt @@ -0,0 +1,27 @@ +# +# Author: Hari Sekhon +# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying LICENSE file +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# RPM PIP Package Requirements +# ============================================================================ # + +GitPython +python-flask +python-humanize +python-jinja2 +python-krbV +python-linecache2 +python-markupsafe +python-thrift +python-xmltodict +python2-bitarray +python2-numpy +python2-psutil From 2906281ac62bf5a9cd73145740dc5a0242131fe1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Feb 2019 01:09:35 +0000 Subject: [PATCH 0065/2779] updated Makefile --- Makefile | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 8c39799f1..821c78c89 100755 --- a/Makefile +++ b/Makefile @@ -108,9 +108,15 @@ python: $(MAKE) parquet-tools; \ fi - # json module built-in to Python >= 2.6, backport not available via pypi - #$(SUDO_PIP) pip install json - + # fixes bug in cffi version detection when installing requests-kerberos + $(SUDO_PIP) pip install --upgrade pip + + # only install pip packages not installed via system packages + #$(SUDO_PIP) pip install --upgrade -r requirements.txt + for pip_module in `sed 's/#.*//; s/[>=].*//; s/-/_/g; /^[[:space:]]*$$/d' requirements.txt`; do \ + python -c "import $$pip_module" || $(SUDO_PIP) pip install "$$pip_module" || exit 1; \ + done + # for impyla $(SUDO_PIP) pip install --upgrade setuptools || : # @@ -125,7 +131,8 @@ python: # sudo su # LD_RUN_PATH=/usr/local/include pip install snappy # - $(SUDO_PIP) pip install --upgrade -r requirements.txt + #$(SUDO_PIP) pip install --upgrade -r requirements.txt + # for ipython-notebook-pyspark.py #$(SUDO_PIP) pip install jinja2 # HiveServer2 @@ -160,6 +167,7 @@ parquet-tools: apk-packages: $(SUDO) apk update $(SUDO) apk add `sed 's/#.*//; /^[[:space:]]*$$/d' setup/apk-packages.txt setup/apk-packages-dev.txt` + for package in `sed 's/#.*//; /^[[:space:]]*$$/d' setup/apk-packages-pip.txt`; do $(SUDO) apk add "$$package" || : ; done if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apk add openjdk8-jre-base; fi # Spark Java Py4J gets java linking error without this if [ -f /lib/libc.musl-x86_64.so.1 ]; then [ -e /lib/ld-linux-x86-64.so.2 ] || ln -sv /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2; fi @@ -181,6 +189,7 @@ apk-packages-remove: apt-packages: $(SUDO) apt-get update $(SUDO) apt-get install -y `sed 's/#.*//; /^[[:space:]]*$$/d' setup/deb-packages.txt setup/deb-packages-dev.txt` + for package in `sed 's/#.*//; /^[[:space:]]*$$/d' setup/deb-packages-pip.txt`; do $(SUDO) apt-get install -y "$$package" || : ; done if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk; fi # for validate_multimedia.py @@ -208,6 +217,7 @@ yum-packages: rpm -q wget || $(SUDO) yum install -y wget rpm -q epel-release || yum install -y epel-release || { wget -t 100 --retry-connrefused -O /tmp/epel.rpm "https://dl.fedoraproject.org/pub/epel/epel-release-latest-`grep -o '[[:digit:]]' /etc/*release | head -n1`.noarch.rpm" && $(SUDO) rpm -ivh /tmp/epel.rpm && rm -f /tmp/epel.rpm; } for x in `sed 's/#.*//; /^[[:space:]]*$$/d' setup/rpm-packages.txt setup/rpm-packages-dev.txt`; do rpm -q $$x || $(SUDO) yum install -y $$x; done + $(SUDO) yum install -y `sed 's/#.*//; /^[[:space:]]*$$/d' setup/rpm-packages-pip.txt` || : if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) yum install -y java; fi # for validate_multimedia.py From ea19a31a4476cb4b8bee405fb301f52e56e52f99 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Feb 2019 01:19:07 +0000 Subject: [PATCH 0066/2779] added setup/deb-packages-pip.txt --- setup/deb-packages-pip.txt | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 setup/deb-packages-pip.txt diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt new file mode 100755 index 000000000..9fe0d6594 --- /dev/null +++ b/setup/deb-packages-pip.txt @@ -0,0 +1,32 @@ +# +# Author: Hari Sekhon +# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying LICENSE file +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# Deb PIP Package Requirements +# ============================================================================ # + +python-avro +python-bitarray +python-dicttoxml +python-flask +python-git +python-happybase +python-humanize +python-jinja2 +python-krbv +python-ldif3 +python-linecache2 +python-numpy +python-psutil +python-sh +python-snappy +python-thrift +python-xmltodict From 1780d63190bc747181b0408e94300aa843b63d68 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Feb 2019 01:23:07 +0000 Subject: [PATCH 0067/2779] added setup/apk-packages-pip.txt --- setup/apk-packages-pip.txt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 setup/apk-packages-pip.txt diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt new file mode 100755 index 000000000..da9724ad7 --- /dev/null +++ b/setup/apk-packages-pip.txt @@ -0,0 +1,20 @@ +# +# Author: Hari Sekhon +# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying LICENSE file +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# Alpine PIP Package Requirements +# ============================================================================ # + +py-dicttoxml +py-flask +py-psutil +py2-jinja2 +py2-numpy From 5ac6c21482e15c87084561521f4a4a22928078a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Feb 2019 08:37:48 +0000 Subject: [PATCH 0068/2779] updated requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 992242e52..b7cdb47ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,4 +35,5 @@ sh>=1.12.14 snakebite[kerberos]>=2.11.0 thrift-sasl>=0.2.1 thrift>=0.9.3 +thriftpy>=0.3.9 xmltodict>=0.10.2 From 1c8de468dd15264af134cccda95f6f3f995e7ad3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Feb 2019 15:08:25 +0000 Subject: [PATCH 0069/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 821c78c89..13dca8544 100755 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ python: # only install pip packages not installed via system packages #$(SUDO_PIP) pip install --upgrade -r requirements.txt for pip_module in `sed 's/#.*//; s/[>=].*//; s/-/_/g; /^[[:space:]]*$$/d' requirements.txt`; do \ - python -c "import $$pip_module" || $(SUDO_PIP) pip install "$$pip_module" || exit 1; \ + python -c "import $$pip_module" || $(SUDO_PIP) pip install --ignore-installed urllib3 "$$pip_module" || exit 1; \ done # for impyla From 458497cbeb2b9ae9767c623579fa345f747f57d7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Feb 2019 15:59:36 +0000 Subject: [PATCH 0070/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 088904d4d..3f5956078 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 088904d4d6049f953494b7921f1f011b30bb1a0f +Subproject commit 3f595607864bea4dd371e195a595b54d504a6851 From 92e3f124e6012a0b1c8b917f0c3e8b6aa3ded4c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 13:07:26 +0000 Subject: [PATCH 0071/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e7b72d5dc..4e6c0c7d5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e7b72d5dc002ee32c7fbe21abed83568b23f881f +Subproject commit 4e6c0c7d5379fa089e7690fb699b2d64c79283fb From 9cf6b5e555a45594098a7a3553bd411a083e1a80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 13:07:27 +0000 Subject: [PATCH 0072/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3f5956078..8815b7d7c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3f595607864bea4dd371e195a595b54d504a6851 +Subproject commit 8815b7d7c12da8fa22d60c7410bc271dc38f5d26 From cdedbb5c7404b5fef3841b7cfb10df9f5e5c7a79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 13:10:34 +0000 Subject: [PATCH 0073/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8815b7d7c..79586140f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8815b7d7c12da8fa22d60c7410bc271dc38f5d26 +Subproject commit 79586140f3bd8e9dcbb7059ac155bfa3d92ea6d7 From 9f02f3afbe242cc264e8076fb183a7199feef1bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 13:10:54 +0000 Subject: [PATCH 0074/2779] added docker quick builds --- Makefile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Makefile b/Makefile index 13dca8544..5685be742 100755 --- a/Makefile +++ b/Makefile @@ -321,3 +321,23 @@ mount: docker-mount .PHONY: push push: git push + +# For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools +.PHONY: docker-alpine +docker-alpine: + docker run -ti -v $$PWD:/py alpine sh /py/bash-tools/exec-interactive.sh 'cd /py && apk add --no-cache make && make build test' + +# For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools +.PHONY: docker-debian +docker-debian: + docker run -ti -v $$PWD:/tools debian /tools/bash-tools/exec-interactive.sh 'cd /tools && apt-get update && apt-get install -y make && make build test' + +# For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools +.PHONY: docker-centos +docker-centos: + docker run -ti -v $$PWD:/tools centos /tools/bash-tools/exec-interactive.sh 'cd /tools && yum install -y make && make build test' + +# For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools +.PHONY: docker-ubuntu +docker-ubuntu: + docker run -ti -v $$PWD:/tools ubuntu /tools/bash-tools/exec-interactive.sh 'cd /tools && apt-get update && apt-get install -y make && make build test' From ce4acb75618f645b6404d25e080fa2edb09444b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 13:29:08 +0000 Subject: [PATCH 0075/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4e6c0c7d5..14300f152 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4e6c0c7d5379fa089e7690fb699b2d64c79283fb +Subproject commit 14300f15231eec5955817537b85a613f408a8765 From 437e11ef04d3fe45a00e29d35af08e84de95f252 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 13:29:09 +0000 Subject: [PATCH 0076/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 79586140f..ff5cc2b66 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 79586140f3bd8e9dcbb7059ac155bfa3d92ea6d7 +Subproject commit ff5cc2b6616f5247665c1d03676aa6932c58d88d From 5740f26050ae620ca2ffd123e8c30fc06df01b89 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 13:29:58 +0000 Subject: [PATCH 0077/2779] updated Makefile --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 5685be742..b450d5acb 100755 --- a/Makefile +++ b/Makefile @@ -325,19 +325,19 @@ push: # For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools .PHONY: docker-alpine docker-alpine: - docker run -ti -v $$PWD:/py alpine sh /py/bash-tools/exec-interactive.sh 'cd /py && apk add --no-cache make && make build test' + bash-tools/docker_mount_build_exec.sh alpine # For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools .PHONY: docker-debian docker-debian: - docker run -ti -v $$PWD:/tools debian /tools/bash-tools/exec-interactive.sh 'cd /tools && apt-get update && apt-get install -y make && make build test' + bash-tools/docker_mount_build_exec.sh debian # For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools .PHONY: docker-centos docker-centos: - docker run -ti -v $$PWD:/tools centos /tools/bash-tools/exec-interactive.sh 'cd /tools && yum install -y make && make build test' + bash-tools/docker_mount_build_exec.sh centos # For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools .PHONY: docker-ubuntu docker-ubuntu: - docker run -ti -v $$PWD:/tools ubuntu /tools/bash-tools/exec-interactive.sh 'cd /tools && apt-get update && apt-get install -y make && make build test' + bash-tools/docker_mount_build_exec.sh ubuntu From 51fce1a20fb2af893e5d135d039edf50b1547679 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 13:38:40 +0000 Subject: [PATCH 0078/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 14300f152..abe0088dd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 14300f15231eec5955817537b85a613f408a8765 +Subproject commit abe0088dd085bed7d1fa7cebea2f7f7da3d0a820 From 3bd454324b6a1dab3ac127c5656a706948fa993e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 13:38:40 +0000 Subject: [PATCH 0079/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ff5cc2b66..034c4932d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ff5cc2b6616f5247665c1d03676aa6932c58d88d +Subproject commit 034c4932d09a460cde47030337a63310171534bc From de66f2111a3aa8ca2d2209e1dcae2148e44a7053 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 14:07:34 +0000 Subject: [PATCH 0080/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index abe0088dd..aa27bcb09 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit abe0088dd085bed7d1fa7cebea2f7f7da3d0a820 +Subproject commit aa27bcb099b68177e52368907bebfffe27660421 From f86cf7bd63d761b5e95c346da95fa16c62cbf2a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 14:08:29 +0000 Subject: [PATCH 0081/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 034c4932d..3382100a4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 034c4932d09a460cde47030337a63310171534bc +Subproject commit 3382100a46bc33b1e5c42ddc592102dc6cb77197 From 2070df3be946bfb8948e457e670efd02b2043291 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 14:08:59 +0000 Subject: [PATCH 0082/2779] updated Makefile --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b450d5acb..a636d20a6 100755 --- a/Makefile +++ b/Makefile @@ -113,9 +113,7 @@ python: # only install pip packages not installed via system packages #$(SUDO_PIP) pip install --upgrade -r requirements.txt - for pip_module in `sed 's/#.*//; s/[>=].*//; s/-/_/g; /^[[:space:]]*$$/d' requirements.txt`; do \ - python -c "import $$pip_module" || $(SUDO_PIP) pip install --ignore-installed urllib3 "$$pip_module" || exit 1; \ - done + @bash-tools/python_pip_install_if_absent.sh requirements.txt # for impyla $(SUDO_PIP) pip install --upgrade setuptools || : From eecd0f5c82efbd4daf050e782de59b177f36c882 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 14:13:36 +0000 Subject: [PATCH 0083/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3382100a4..a3ce5751f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3382100a46bc33b1e5c42ddc592102dc6cb77197 +Subproject commit a3ce5751f6c797f300db7b2f82e5d918cd811d8f From 59fb52c044092b770122011271c8445ed6ed4a7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 15:17:16 +0000 Subject: [PATCH 0084/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a3ce5751f..d838dfdd3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a3ce5751f6c797f300db7b2f82e5d918cd811d8f +Subproject commit d838dfdd33db23e7f2a91d39d89f19d9d42b47ae From 72e358b93d83487e8663ef24751b5f92a6ad5e46 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 15:17:53 +0000 Subject: [PATCH 0085/2779] updated Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a636d20a6..57c3e81bd 100755 --- a/Makefile +++ b/Makefile @@ -113,7 +113,8 @@ python: # only install pip packages not installed via system packages #$(SUDO_PIP) pip install --upgrade -r requirements.txt - @bash-tools/python_pip_install_if_absent.sh requirements.txt + #@bash-tools/python_pip_install_if_absent.sh requirements.txt + $(SUDO_PIP) pip install -r requirements.txt # for impyla $(SUDO_PIP) pip install --upgrade setuptools || : From 035b51de9ea1d551272dc5fe54b07ac26f06d3d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 15:39:25 +0000 Subject: [PATCH 0086/2779] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 57c3e81bd..42767d161 100755 --- a/Makefile +++ b/Makefile @@ -113,8 +113,8 @@ python: # only install pip packages not installed via system packages #$(SUDO_PIP) pip install --upgrade -r requirements.txt - #@bash-tools/python_pip_install_if_absent.sh requirements.txt - $(SUDO_PIP) pip install -r requirements.txt + #$(SUDO_PIP) pip install -r requirements.txt + @bash-tools/python_pip_install_if_absent.sh requirements.txt # for impyla $(SUDO_PIP) pip install --upgrade setuptools || : From 5031a55cd22e944d802a7285fc2d6d49412a5eb7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 15:39:32 +0000 Subject: [PATCH 0087/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index aa27bcb09..f68000004 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit aa27bcb099b68177e52368907bebfffe27660421 +Subproject commit f68000004d3d8a8779cf94046948568bde306c7b From 81e030330f225ae20c002ebde0c982ded5fd5a5b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 15:39:33 +0000 Subject: [PATCH 0088/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d838dfdd3..2cfdd04e8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d838dfdd33db23e7f2a91d39d89f19d9d42b47ae +Subproject commit 2cfdd04e8ac5299e2d6f56c86d2f102ec711a2c8 From 0b0e583c0b6dcdaf0595fdcbac1dc45b1518a169 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 21:26:30 +0000 Subject: [PATCH 0089/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f68000004..a0da87e5b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f68000004d3d8a8779cf94046948568bde306c7b +Subproject commit a0da87e5b30f43aae49b8a94c6e71784d851ee7a From 465292e28a1048df637bda38d34d2c00cd902ab1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Feb 2019 21:26:30 +0000 Subject: [PATCH 0090/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2cfdd04e8..9848cdfa5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2cfdd04e8ac5299e2d6f56c86d2f102ec711a2c8 +Subproject commit 9848cdfa5c9f515421f7536f66df087f65c3c385 From fa33ca4545d6bb2e36305f1f880935c53e913239 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Feb 2019 21:37:03 +0000 Subject: [PATCH 0091/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9848cdfa5..221ff45a6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9848cdfa5c9f515421f7536f66df087f65c3c385 +Subproject commit 221ff45a6dfaf2effd04920de8ae0b160bd58b3b From 1ef82fd2dba1bd81525593c02b0c0850ae742715 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 11:07:45 +0000 Subject: [PATCH 0092/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a0da87e5b..83e23886b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a0da87e5b30f43aae49b8a94c6e71784d851ee7a +Subproject commit 83e23886b843db4375ce7f84ea7b7a30c953a6ff From c473542c35dbc68ad90d025d54d913c9a2ee1e71 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 11:07:45 +0000 Subject: [PATCH 0093/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 221ff45a6..857dad06e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 221ff45a6dfaf2effd04920de8ae0b160bd58b3b +Subproject commit 857dad06e7d443e178c29148381effc69b9e8bca From 5706d5fa89a7761211c3fdd858afecf68271ca8e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 11:27:52 +0000 Subject: [PATCH 0094/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 83e23886b..6fd3910ed 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 83e23886b843db4375ce7f84ea7b7a30c953a6ff +Subproject commit 6fd3910ed2926d05851e411040baf899ee0a8a59 From 32598f8a52d426e9f5603fafb18f7b7f551531a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 11:27:52 +0000 Subject: [PATCH 0095/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 857dad06e..1c08878f7 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 857dad06e7d443e178c29148381effc69b9e8bca +Subproject commit 1c08878f7f84f90d8816f0f0b5eb78a07f6c9a5b From db79bdc5bfed6b8c8d7cf95994407804fb4abb3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 11:40:34 +0000 Subject: [PATCH 0096/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6fd3910ed..74ae3b028 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6fd3910ed2926d05851e411040baf899ee0a8a59 +Subproject commit 74ae3b0285328348f84df2b4aaba9bdd6d89441a From f1da82b3420a16ca1376dcf0d19356ab612522ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 11:40:34 +0000 Subject: [PATCH 0097/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1c08878f7..d4c8719df 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1c08878f7f84f90d8816f0f0b5eb78a07f6c9a5b +Subproject commit d4c8719dfa458a115c59266ebb327365a7afcccc From a64474962c02afb9f462d82e8539833eb15e0bc6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 11:48:14 +0000 Subject: [PATCH 0098/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 74ae3b028..82dcbe49d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 74ae3b0285328348f84df2b4aaba9bdd6d89441a +Subproject commit 82dcbe49d5d14ebb8c99a2d6055f433035a69c63 From da1075c96e9d35535956f33f96b7da17663d58af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 11:48:14 +0000 Subject: [PATCH 0099/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d4c8719df..a72b980f0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d4c8719dfa458a115c59266ebb327365a7afcccc +Subproject commit a72b980f039a64ed5c2ce37afabfda42498181cd From 6e871af7b13d06df4007858a95dddf6adf687710 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 20:31:22 +0000 Subject: [PATCH 0100/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 82dcbe49d..daa9b5c29 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 82dcbe49d5d14ebb8c99a2d6055f433035a69c63 +Subproject commit daa9b5c295e24982402ddc1cdb0c56b45e2e7139 From 75badd11bd562a7d4bbaf27e04c0b81833f68bd5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 20:31:22 +0000 Subject: [PATCH 0101/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a72b980f0..ca7d2e605 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a72b980f039a64ed5c2ce37afabfda42498181cd +Subproject commit ca7d2e605bf9bb8b4f25cb8fcd307851fc392fe5 From d5e48ab98d06f6413cc659767f17a963ab222b0c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 23:24:09 +0000 Subject: [PATCH 0102/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index daa9b5c29..0ff81468f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit daa9b5c295e24982402ddc1cdb0c56b45e2e7139 +Subproject commit 0ff81468ffc2c6a9fdd6e701cb0ab73be3b35896 From 70fef3841dcec918215e435a0b19401370f07c3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Feb 2019 23:24:19 +0000 Subject: [PATCH 0103/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ca7d2e605..ddf814882 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ca7d2e605bf9bb8b4f25cb8fcd307851fc392fe5 +Subproject commit ddf814882e990d46a80e83a1b544d7e5a7190b17 From dec06d759d83ae963442afbe2f422a365bd7fca2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Feb 2019 16:45:04 +0000 Subject: [PATCH 0104/2779] updated .travis.yml --- .travis.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index e64530f13..9a3819dd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,8 +54,13 @@ script: after_success: - coveralls - - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/pytools/trigger/0e69c39f-ea1b-43c7-a97d-cef1252f1400/ - - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/alpine-github/trigger/8d9cb826-48df-439c-8c20-1975713064fc/ - - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/centos-github/trigger/316d1158-7ffb-49a4-a7bd-8e5456ba2d15/ - - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/debian-github/trigger/0cfb3fe7-2028-494b-a43b-068435e6a2b3/ - - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/ubuntu-github/trigger/abd4dbf0-14bc-454f-9cde-081ec014bc48/ + # PyTools + - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/d470810b-9a44-4abc-92cc-c903a6afd962/trigger/0e69c39f-ea1b-43c7-a97d-cef1252f1400/call/ + # Alpine Github + - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/df816f2a-9407-4f1b-8b51-39615d784e65/trigger/8d9cb826-48df-439c-8c20-1975713064fc/call/ + # Debian Github + - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/439eff84-50c7-464a-a49e-0ac0bf1a9a43/trigger/0cfb3fe7-2028-494b-a43b-068435e6a2b3/call/ + # CentOS Github + - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/efba1846-5a9e-470a-92f8-69edc1232ba0/trigger/316d1158-7ffb-49a4-a7bd-8e5456ba2d15/call/ + # Ubuntu Github + - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/8b3dc094-d4ca-4c92-861e-1e842b5fac42/trigger/abd4dbf0-14bc-454f-9cde-081ec014bc48/call/ From fb6209a64e77620f849add9168a93d2eb5be2ed8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Feb 2019 16:50:27 +0000 Subject: [PATCH 0105/2779] updated .travis.yml --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9a3819dd8..6b08a8c1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,12 +55,12 @@ script: after_success: - coveralls # PyTools - - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/d470810b-9a44-4abc-92cc-c903a6afd962/trigger/0e69c39f-ea1b-43c7-a97d-cef1252f1400/call/ + - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/d470810b-9a44-4abc-92cc-c903a6afd962/trigger/0e69c39f-ea1b-43c7-a97d-cef1252f1400/call/ # Alpine Github - - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/df816f2a-9407-4f1b-8b51-39615d784e65/trigger/8d9cb826-48df-439c-8c20-1975713064fc/call/ + - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/df816f2a-9407-4f1b-8b51-39615d784e65/trigger/8d9cb826-48df-439c-8c20-1975713064fc/call/ # Debian Github - - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/439eff84-50c7-464a-a49e-0ac0bf1a9a43/trigger/0cfb3fe7-2028-494b-a43b-068435e6a2b3/call/ + - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/439eff84-50c7-464a-a49e-0ac0bf1a9a43/trigger/0cfb3fe7-2028-494b-a43b-068435e6a2b3/call/ # CentOS Github - - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/efba1846-5a9e-470a-92f8-69edc1232ba0/trigger/316d1158-7ffb-49a4-a7bd-8e5456ba2d15/call/ + - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/efba1846-5a9e-470a-92f8-69edc1232ba0/trigger/316d1158-7ffb-49a4-a7bd-8e5456ba2d15/call/ # Ubuntu Github - - curl --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/8b3dc094-d4ca-4c92-861e-1e842b5fac42/trigger/abd4dbf0-14bc-454f-9cde-081ec014bc48/call/ + - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/8b3dc094-d4ca-4c92-861e-1e842b5fac42/trigger/abd4dbf0-14bc-454f-9cde-081ec014bc48/call/ From afb0f9e85e6d5000e7819fe5d0d4f5f56212fe70 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Feb 2019 22:10:45 +0000 Subject: [PATCH 0106/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0ff81468f..7f81defb6 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0ff81468ffc2c6a9fdd6e701cb0ab73be3b35896 +Subproject commit 7f81defb6f13590d01bd76c88e4eab192bfaea49 From 781274654c729bf73cd6779eab4ffbc5076ddc55 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Feb 2019 22:10:45 +0000 Subject: [PATCH 0107/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ddf814882..b26ee694c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ddf814882e990d46a80e83a1b544d7e5a7190b17 +Subproject commit b26ee694c4be847590826fe6814f1716cee1a787 From d041bb3ab28f50b02b1b735d8b31d017998905fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Feb 2019 23:08:13 +0000 Subject: [PATCH 0108/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b26ee694c..314551713 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b26ee694c4be847590826fe6814f1716cee1a787 +Subproject commit 314551713f838fdddc74a88142f23ce7a6d7c994 From cdf96147e27c2ed734d0c7356cb092bb196754a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Feb 2019 23:33:17 +0000 Subject: [PATCH 0109/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 314551713..2486fd04f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 314551713f838fdddc74a88142f23ce7a6d7c994 +Subproject commit 2486fd04fe555ddd7b26536cc47fbef31609289e From eeb31e6cb116beed14274cd6d3e685b048a92f0e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 00:20:14 +0000 Subject: [PATCH 0110/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7f81defb6..f5aae3578 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7f81defb6f13590d01bd76c88e4eab192bfaea49 +Subproject commit f5aae3578b720bc7dd1bc62bc9777fc8f5f9a598 From 5748c07afe18d4cf3d52d2070bac8688988b2ef2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 00:20:14 +0000 Subject: [PATCH 0111/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2486fd04f..ad943461c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2486fd04fe555ddd7b26536cc47fbef31609289e +Subproject commit ad943461c73e7b91e8552df3e5c78b4f8f879e02 From dc78305521e868760f29d2e6b5a8f29170ab07c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 00:45:56 +0000 Subject: [PATCH 0112/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f5aae3578..1d9e1a5fd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f5aae3578b720bc7dd1bc62bc9777fc8f5f9a598 +Subproject commit 1d9e1a5fd737d34a89459f2deb9ba941aac82949 From 8337dd244a4dd33274d8f5c6a0cfdd1fb20092ac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 01:35:39 +0000 Subject: [PATCH 0113/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1d9e1a5fd..d8801b503 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1d9e1a5fd737d34a89459f2deb9ba941aac82949 +Subproject commit d8801b503f07a0412ae02056ae6f7495b3f5c394 From 2d9f255f6e70b9ad43390be0d379d28fcbf999fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 01:35:39 +0000 Subject: [PATCH 0114/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ad943461c..e83ea3e80 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ad943461c73e7b91e8552df3e5c78b4f8f879e02 +Subproject commit e83ea3e809aeb517a3b8638d536b085a830596fd From 13fb4b8705a81779d0e26a83f3d8eb9c9090b2e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 01:41:25 +0000 Subject: [PATCH 0115/2779] updated rpm-packages.txt --- setup/rpm-packages.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index ba28993d9..fec3a42af 100755 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -33,4 +33,4 @@ https-tools # needed to build python-snappy for avro module snappy-devel -#which java || $(SUDO) yum install -y java +java From d1252627c4f2d701775a2aad0ffe64d65248ddbc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 01:43:27 +0000 Subject: [PATCH 0116/2779] updated Makefile --- Makefile | 195 +++++++------------------------------------------------ 1 file changed, 22 insertions(+), 173 deletions(-) diff --git a/Makefile b/Makefile index 42767d161..2ecfb2bc2 100755 --- a/Makefile +++ b/Makefile @@ -12,47 +12,6 @@ # Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules to non-system python otherwise they're not found by programs. # Better than modifying $PATH to put /usr/bin first which is likely to affect many other things including potentially not finding the perlbrew installation first -# silences warnings but breaks ifneq '$(VIRTUAL_ENV)$(CONDA_DEFAULT_ENV)$(TRAVIS)' '' -# -#ifndef VIRTUAL_ENV -# VIRTUAL_ENV = '' -#endif -#ifndef CONDA_DEFAULT_ENV -# CONDA_DEFAULT_ENV = '' -#endif -#ifndef TRAVIS -# TRAVIS = '' -#endif - -SUDO := sudo -H -SUDO_PIP := sudo -H - -ifdef VIRTUAL_ENV - # breaks as command before first target - #$(info VIRTUAL_ENV environment variable detected, not using sudo) - SUDO_PIP := -endif -ifdef CONDA_DEFAULT_ENV - #$(info CONDA_DEFAULT_ENV environment variable detected, not using sudo) - SUDO_PIP := -endif -#ifdef TRAVIS - # this breaks before first target - #$(info TRAVIS environment variable detected, not using sudo) -# SUDO_PIP := -#endif - -# must come after to reset SUDO_PIP to blank if root -# EUID / UID not exported in Make -# USER not populated in Docker -ifeq '$(shell id -u)' '0' - #$(info UID = 0 detected, not using sudo) - SUDO := - SUDO_PIP := -endif - -PARQUET_VERSION=1.5.0 - # =================== # bootstrap commands: @@ -70,30 +29,28 @@ PARQUET_VERSION=1.5.0 # =================== +ifneq ("$(wildcard bash-tools/Makefile.in)", "") + include bash-tools/Makefile.in +endif + +PARQUET_VERSION=1.5.0 + .PHONY: build build: @echo ============= @echo DevOps Python Tools Build @echo ============= - $(MAKE) common + $(MAKE) init + if [ -z "$(CPANM)" ]; then make; exit $?; fi + $(MAKE) system-packages + if type apk 2&>/dev/null; then $(MAKE) apk-packages-extra; fi + if type apt-get 2&>/dev/null; then $(MAKE) apt-packages-extra; fi $(MAKE) python -.PHONY: system-packages -system-packages: - if [ -x /sbin/apk ]; then $(MAKE) apk-packages; fi - if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages; fi - if [ -x /usr/bin/yum ]; then $(MAKE) yum-packages; fi - if [ -x /usr/local/bin/brew -a `uname` = Darwin ]; then $(MAKE) homebrew-packages; fi - -.PHONY: common -common: system-packages submodules - : - -.PHONY: submodules -submodules: - git submodule init - git submodule update --recursive +.PHONY: init +init: + git submodule update --init --recursive .PHONY: python python: @@ -108,16 +65,13 @@ python: $(MAKE) parquet-tools; \ fi - # fixes bug in cffi version detection when installing requests-kerberos - $(SUDO_PIP) pip install --upgrade pip - # only install pip packages not installed via system packages #$(SUDO_PIP) pip install --upgrade -r requirements.txt #$(SUDO_PIP) pip install -r requirements.txt @bash-tools/python_pip_install_if_absent.sh requirements.txt # for impyla - $(SUDO_PIP) pip install --upgrade setuptools || : + #$(SUDO_PIP) pip install --upgrade setuptools || : # # snappy may fail to install on Mac not finding snappy-c.h - workaround: # @@ -139,7 +93,7 @@ python: # Impala #$(SUDO_PIP) pip install impyla # must downgrade happybase library to work on Python 2.6 - if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi + #if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi # Python >= 2.7 - won't build on 2.6, handle separately and accept failure $(SUDO_PIP) pip install "ipython[notebook]" || : @@ -151,10 +105,6 @@ python: @echo @echo 'BUILD SUCCESSFUL (pytools)' -.PHONY: quick -quick: - QUICK=1 $(MAKE) - .PHONY: parquet-tools parquet-tools: if ! [ -d "parquet-tools-$(PARQUET_VERSION)" ]; then \ @@ -162,15 +112,16 @@ parquet-tools: unzip "parquet-tools-$(PARQUET_VERSION)-bin.zip"; \ fi -.PHONY: apk-packages -apk-packages: - $(SUDO) apk update - $(SUDO) apk add `sed 's/#.*//; /^[[:space:]]*$$/d' setup/apk-packages.txt setup/apk-packages-dev.txt` - for package in `sed 's/#.*//; /^[[:space:]]*$$/d' setup/apk-packages-pip.txt`; do $(SUDO) apk add "$$package" || : ; done +.PHONY: apk-packages-extra +apk-packages-extra: if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apk add openjdk8-jre-base; fi # Spark Java Py4J gets java linking error without this if [ -f /lib/libc.musl-x86_64.so.1 ]; then [ -e /lib/ld-linux-x86-64.so.2 ] || ln -sv /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2; fi +.PHONY: apt-packages-extra +apt-packages-extra: + if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk; fi + # for validate_multimedia.py # available in Alpine 2.6, 2.7 and 3.x .PHONY: apk-packages-multimedia @@ -178,19 +129,6 @@ apk-packages-multimedia: $(SUDO) apk update $(SUDO) apk add ffmpeg -.PHONY: apk-packages-remove -apk-packages-remove: - cd pylib && $(MAKE) apk-packages-remove - $(SUDO) apk del `sed 's/#.*//; /^[[:space:]]*$$/d' < setup/apk-packages-dev.txt` || : - $(SUDO) rm -fr /var/cache/apk/* - -.PHONY: apt-packages -apt-packages: - $(SUDO) apt-get update - $(SUDO) apt-get install -y `sed 's/#.*//; /^[[:space:]]*$$/d' setup/deb-packages.txt setup/deb-packages-dev.txt` - for package in `sed 's/#.*//; /^[[:space:]]*$$/d' setup/deb-packages-pip.txt`; do $(SUDO) apt-get install -y "$$package" || : ; done - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk; fi - # for validate_multimedia.py # Ubuntu 16.04 Xenial onwards, not available in Ubuntu 14.04 Trusty # Debian 9 Stretch onwards, not available in Debian 8 Jessie @@ -199,37 +137,12 @@ apt-packages-multimedia: $(SUDO) apt-get update $(SUDO) apt-get install -y --no-install-recommends ffmpeg -.PHONY: apt-packages-remove -apt-packages-remove: - cd pylib && $(MAKE) apt-packages-remove - $(SUDO) apt-get purge -y `sed 's/#.*//; /^[[:space:]]*$$/d' < setup/deb-packages-dev.txt` - -.PHONY: homebrew-packages -homebrew-packages: - # Sudo is not required as running Homebrew as root is extremely dangerous and no longer supported as Homebrew does not drop privileges on installation you would be giving all build scripts full access to your system - # Fails if any of the packages are already installed, ignore and continue - if it's a problem the latest build steps will fail with missing headers - brew install `sed 's/#.*//; /^[[:space:]]*$$/d' setup/brew-packages.txt` || : - -.PHONY: yum-packages -yum-packages: - # python-pip requires EPEL, so try to get the correct EPEL rpm - rpm -q wget || $(SUDO) yum install -y wget - rpm -q epel-release || yum install -y epel-release || { wget -t 100 --retry-connrefused -O /tmp/epel.rpm "https://dl.fedoraproject.org/pub/epel/epel-release-latest-`grep -o '[[:digit:]]' /etc/*release | head -n1`.noarch.rpm" && $(SUDO) rpm -ivh /tmp/epel.rpm && rm -f /tmp/epel.rpm; } - for x in `sed 's/#.*//; /^[[:space:]]*$$/d' setup/rpm-packages.txt setup/rpm-packages-dev.txt`; do rpm -q $$x || $(SUDO) yum install -y $$x; done - $(SUDO) yum install -y `sed 's/#.*//; /^[[:space:]]*$$/d' setup/rpm-packages-pip.txt` || : - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) yum install -y java; fi - # for validate_multimedia.py .PHONY: yum-packages-multimedia yum-packages-multimedia: @echo "This requires 3rd party rpm repos which could result in rpm hell, please handle this manually yourself so you understand what you're doing" exit 1 -.PHONY: yum-packages-remove -yum-packages-remove: - cd pylib && $(MAKE) yum-packages-remove - for x in `sed 's/#.*//; /^[[:space:]]*$$/d' < setup/rpm-packages-dev.txt`; do rpm -q $$x && $(SUDO) yum remove -y $$x; done - .PHONY: jython jython: if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi @@ -237,10 +150,6 @@ jython: if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi sh jython_install.sh -.PHONY: sonar -sonar: - sonar-scanner - .PHONY: test-lib test-lib: cd pylib && $(MAKE) test @@ -262,26 +171,6 @@ test2: install: @echo "No installation needed, just add '$(PWD)' to your \$$PATH" -.PHONY: update -update: update2 build - : - -.PHONY: update2 -update2: update-no-recompile - : - -.PHONY: update-no-recompile -update-no-recompile: - git pull - git submodule update --init --recursive - -.PHONY: update-submodules -update-submodules: - git submodule update --init --remote -.PHONY: updatem -updatem: update-submodules - : - .PHONY: clean clean: cd pylib && $(MAKE) clean @@ -300,43 +189,3 @@ deep-clean: clean spark-deps: rm -vf spark-deps.zip zip spark-deps.zip pylib - -.PHONY: docker-run -docker-run: - docker run -ti --rm harisekhon/pytools ${ARGS} - -.PHONY: run -run: - $(MAKE) docker-run - -.PHONY: docker-mount -docker-mount: - docker run -ti --rm -v $$PWD:/py harisekhon/pytools bash -c "cd /py; exec bash" - -.PHONY: mount -mount: docker-mount - : - -.PHONY: push -push: - git push - -# For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools -.PHONY: docker-alpine -docker-alpine: - bash-tools/docker_mount_build_exec.sh alpine - -# For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools -.PHONY: docker-debian -docker-debian: - bash-tools/docker_mount_build_exec.sh debian - -# For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools -.PHONY: docker-centos -docker-centos: - bash-tools/docker_mount_build_exec.sh centos - -# For quick testing only - for actual Dockerfile builds see https://hub.docker.com/r/harisekhon/tools -.PHONY: docker-ubuntu -docker-ubuntu: - bash-tools/docker_mount_build_exec.sh ubuntu From 4581e5080fe343dc7786bd2926d2443a8375a41b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 01:59:18 +0000 Subject: [PATCH 0117/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d8801b503..491605bbc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d8801b503f07a0412ae02056ae6f7495b3f5c394 +Subproject commit 491605bbc57bfff9a20453566dcba7cca0e6358b From 1210448fce3e593a7fb9af8a079425db6cdc424e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 01:59:18 +0000 Subject: [PATCH 0118/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e83ea3e80..e78ed587a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e83ea3e809aeb517a3b8638d536b085a830596fd +Subproject commit e78ed587a1837909e2032808a654a496c9447264 From 4b498e7c1c1b047dabf612d7d777ab97a36be148 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 07:50:51 +0000 Subject: [PATCH 0119/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 491605bbc..acbf0fc30 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 491605bbc57bfff9a20453566dcba7cca0e6358b +Subproject commit acbf0fc30ca03c027d13c7990003444796ba1ba6 From 1d394cc1ad6225907901dde919173c8ee445fb82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 11:01:42 +0000 Subject: [PATCH 0120/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index acbf0fc30..79ee31aa6 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit acbf0fc30ca03c027d13c7990003444796ba1ba6 +Subproject commit 79ee31aa655ccec613de44b65d559ecf5f01342a From f0699bdf36c4dc0f77916296456441cac7116bb8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Feb 2019 11:01:42 +0000 Subject: [PATCH 0121/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e78ed587a..60147c760 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e78ed587a1837909e2032808a654a496c9447264 +Subproject commit 60147c76051f7a26265231bb0d171e7f44b004ff From 709b24c89d3cc498a850c629126bb4d84137dcf8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Feb 2019 11:47:10 +0000 Subject: [PATCH 0122/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 79ee31aa6..12ca7c69a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 79ee31aa655ccec613de44b65d559ecf5f01342a +Subproject commit 12ca7c69ad37b6fb636903492155f226a38162b1 From 7b0f656616e3962bf9f51199790f15700288ccca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Feb 2019 11:47:10 +0000 Subject: [PATCH 0123/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 60147c760..3ce72963b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 60147c76051f7a26265231bb0d171e7f44b004ff +Subproject commit 3ce72963b86d2a8338524189ca52f3208103a7b4 From 9be0bfe1a0b43e4c278608a69fa826b90535d052 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Feb 2019 11:47:19 +0000 Subject: [PATCH 0124/2779] updated .travis.yml --- .travis.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b08a8c1c..fcffb06e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,13 +54,3 @@ script: after_success: - coveralls - # PyTools - - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/d470810b-9a44-4abc-92cc-c903a6afd962/trigger/0e69c39f-ea1b-43c7-a97d-cef1252f1400/call/ - # Alpine Github - - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/df816f2a-9407-4f1b-8b51-39615d784e65/trigger/8d9cb826-48df-439c-8c20-1975713064fc/call/ - # Debian Github - - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/439eff84-50c7-464a-a49e-0ac0bf1a9a43/trigger/0cfb3fe7-2028-494b-a43b-068435e6a2b3/call/ - # CentOS Github - - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/efba1846-5a9e-470a-92f8-69edc1232ba0/trigger/316d1158-7ffb-49a4-a7bd-8e5456ba2d15/call/ - # Ubuntu Github - - curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/8b3dc094-d4ca-4c92-861e-1e842b5fac42/trigger/abd4dbf0-14bc-454f-9cde-081ec014bc48/call/ From c45f6fbf535c5c51a4e877eb6f117c63be412b97 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Feb 2019 11:47:41 +0000 Subject: [PATCH 0125/2779] updated Makefile --- Makefile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Makefile b/Makefile index 2ecfb2bc2..6edab9cec 100755 --- a/Makefile +++ b/Makefile @@ -189,3 +189,16 @@ deep-clean: clean spark-deps: rm -vf spark-deps.zip zip spark-deps.zip pylib + +.PHONY: dockerhub +dockerhub: + # PyTools + curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/d470810b-9a44-4abc-92cc-c903a6afd962/trigger/0e69c39f-ea1b-43c7-a97d-cef1252f1400/call/ + # Alpine Github + curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/df816f2a-9407-4f1b-8b51-39615d784e65/trigger/8d9cb826-48df-439c-8c20-1975713064fc/call/ + # Debian Github + curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/439eff84-50c7-464a-a49e-0ac0bf1a9a43/trigger/0cfb3fe7-2028-494b-a43b-068435e6a2b3/call/ + # CentOS Github + curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/efba1846-5a9e-470a-92f8-69edc1232ba0/trigger/316d1158-7ffb-49a4-a7bd-8e5456ba2d15/call/ + # Ubuntu Github + curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/8b3dc094-d4ca-4c92-861e-1e842b5fac42/trigger/abd4dbf0-14bc-454f-9cde-081ec014bc48/call/ From 5f7b407a1b7706a1fbe3db02ac65bab3d9aec0b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 25 Feb 2019 11:21:07 +0000 Subject: [PATCH 0126/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6edab9cec..af9711901 100755 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ build: @echo ============= $(MAKE) init - if [ -z "$(CPANM)" ]; then make; exit $?; fi + if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages if type apk 2&>/dev/null; then $(MAKE) apk-packages-extra; fi if type apt-get 2&>/dev/null; then $(MAKE) apt-packages-extra; fi From f67916dcf8f8da9c2bb76e10e829fe5ef8b51e91 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 25 Feb 2019 12:13:48 +0000 Subject: [PATCH 0127/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 12ca7c69a..870524264 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 12ca7c69ad37b6fb636903492155f226a38162b1 +Subproject commit 8705242646549b746bc3a107ded6c8379d0d01dd From 7e761949aa0f17100e3a4680b573572465b12f48 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 25 Feb 2019 12:13:48 +0000 Subject: [PATCH 0128/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3ce72963b..bbb944db3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3ce72963b86d2a8338524189ca52f3208103a7b4 +Subproject commit bbb944db3e32d6d0b588f01d41e287cf8718649f From f44f1c977bff6406d4fa21d3860257e4282448eb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 25 Feb 2019 12:14:53 +0000 Subject: [PATCH 0129/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index af9711901..2898b573a 100755 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ build: $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi - $(MAKE) system-packages + $(MAKE) system-packages-python if type apk 2&>/dev/null; then $(MAKE) apk-packages-extra; fi if type apt-get 2&>/dev/null; then $(MAKE) apt-packages-extra; fi $(MAKE) python From 4582d498a6d5c1212878fc89794f0163aa9d34d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 25 Feb 2019 13:31:54 +0000 Subject: [PATCH 0130/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 870524264..e94b47800 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8705242646549b746bc3a107ded6c8379d0d01dd +Subproject commit e94b4780022f847d8bf959b90da795a65ce25c3a From 6b716f590256514468d7601b227cf7d18b856bdb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 25 Feb 2019 13:31:54 +0000 Subject: [PATCH 0131/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index bbb944db3..5f5290910 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit bbb944db3e32d6d0b588f01d41e287cf8718649f +Subproject commit 5f52909106a6429c5a2f27f3de7e4f2b088cbc7d From 03b64f98e80391d02eaa01ce6a8ebf902773f166 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 00:27:50 +0000 Subject: [PATCH 0132/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e94b47800..4aaac4f7a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e94b4780022f847d8bf959b90da795a65ce25c3a +Subproject commit 4aaac4f7acd13b5f2d3cef79bf01c8c844a3ac36 From 08807c787d5dddbc77a2943fb97baa9c636a4885 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 00:27:50 +0000 Subject: [PATCH 0133/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5f5290910..dbc2428af 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5f52909106a6429c5a2f27f3de7e4f2b088cbc7d +Subproject commit dbc2428af7dadbf15ccecb0178ded19e37bacb2e From 60c9490670c309f32c6e3758b629a57ef2a031a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 14:39:37 +0000 Subject: [PATCH 0134/2779] added rpm-packages-optional.txt, deb-packages-optional.txt --- setup/deb-packages-optional.txt | 16 ++++++++++++++++ setup/rpm-packages-optional.txt | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100755 setup/deb-packages-optional.txt create mode 100755 setup/rpm-packages-optional.txt diff --git a/setup/deb-packages-optional.txt b/setup/deb-packages-optional.txt new file mode 100755 index 000000000..038ae8802 --- /dev/null +++ b/setup/deb-packages-optional.txt @@ -0,0 +1,16 @@ +# +# Author: Hari Sekhon +# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying LICENSE file +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# Deb Optional Package Requirements +# ============================================================================ # + +yamllint diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt new file mode 100755 index 000000000..f94b6019d --- /dev/null +++ b/setup/rpm-packages-optional.txt @@ -0,0 +1,16 @@ +# +# Author: Hari Sekhon +# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying LICENSE file +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# RPM Optional Package Requirements +# ============================================================================ # + +yamllint From cf9291405abfa0c26202cd1b23d1f76ebfd1c8cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 14:40:16 +0000 Subject: [PATCH 0135/2779] updated requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b7cdb47ec..6e7ab2110 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,3 +37,4 @@ thrift-sasl>=0.2.1 thrift>=0.9.3 thriftpy>=0.3.9 xmltodict>=0.10.2 +yamllint>=1.15.0 From 259fbc97a8877ca26a6561352e49ca20e2d54a90 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 14:50:08 +0000 Subject: [PATCH 0136/2779] added .yamllint --- .yamllint | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .yamllint diff --git a/.yamllint b/.yamllint new file mode 100644 index 000000000..d3ed7f7ac --- /dev/null +++ b/.yamllint @@ -0,0 +1,33 @@ +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +--- + +extends: default + +rules: + # error too many spaces after colon + colons: disable + + # warning missing starting space in comment + comments: disable + + # warning comment not indented like content + comments-indentation: disable + + # warning missing document start "---" + document-start: disable + + # error line too long (94 > 80 characters) + line-length: disable From 366b26b78ce3ec8e725092da50defab70f1b38cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 14:52:00 +0000 Subject: [PATCH 0137/2779] added test_yamllint.sh --- tests/test_yamllint.sh | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 tests/test_yamllint.sh diff --git a/tests/test_yamllint.sh b/tests/test_yamllint.sh new file mode 100755 index 000000000..411432530 --- /dev/null +++ b/tests/test_yamllint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-02-26 14:40:53 +0000 (Tue, 26 Feb 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cfg="$srcdir/../.yamllint" + +cd "$srcdir/.."; + +. ./tests/utils.sh + +section "Running Yaml lint" + +yamllint -c "$cfg" . + +echo "=======" +echo "SUCCESS" +echo "=======" + +echo +echo From b13fa2b0e68eafcfb414421a0ff329ba2a9e569c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 14:52:32 +0000 Subject: [PATCH 0138/2779] updated all.sh --- tests/all.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/all.sh b/tests/all.sh index 140d28926..ee868a3bb 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -30,9 +30,11 @@ section "Running PyTools ALL" cd "$srcdir/.."; bash-tools/all.sh -exit 0 - -bash-tools/run_tests.sh +tests/test_yamllint.sh # do help afterwards for Spark to be downloaded, and then help will find and use downloaded spark for SPARK_HOME tests/help.sh + +exit 0 + +bash-tools/run_tests.sh From 36378ac7334b52cd861326962b2e8535024f0f14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 14:54:58 +0000 Subject: [PATCH 0139/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4aaac4f7a..56c74169c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4aaac4f7acd13b5f2d3cef79bf01c8c844a3ac36 +Subproject commit 56c74169c5904b159505bbc38816dceb037daf57 From f1ec7bc07e47f60614a75b6f610f94f2541f5e33 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 14:54:58 +0000 Subject: [PATCH 0140/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index dbc2428af..180c27175 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit dbc2428af7dadbf15ccecb0178ded19e37bacb2e +Subproject commit 180c271755adb13d96fe7a0a2703478e3e0a3429 From 84bb9246a590595152c83937e5ae84d81a14f3e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 15:18:04 +0000 Subject: [PATCH 0141/2779] updated all.sh --- tests/all.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/all.sh b/tests/all.sh index ee868a3bb..8206b4347 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -33,8 +33,9 @@ bash-tools/all.sh tests/test_yamllint.sh # do help afterwards for Spark to be downloaded, and then help will find and use downloaded spark for SPARK_HOME -tests/help.sh - exit 0 +# pyspark not found +tests/help.sh + bash-tools/run_tests.sh From cac7e2375067c6a81800a45f0345bc6f7cca2f27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 15:36:10 +0000 Subject: [PATCH 0142/2779] Update brew-packages.txt --- setup/brew-packages.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index abbbc3439..aa5eb8f28 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -16,3 +16,4 @@ openssl wget +yamllint From 464a6c2887e8fac012c5e0f22d01866250c6af70 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 16:06:12 +0000 Subject: [PATCH 0143/2779] updated tests/test_anonymize.sh --- tests/test_anonymize.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 8b731b6e9..3f2238f5e 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -418,6 +418,12 @@ dest[107]="hdfs:///" src[108]="hdfs:///user/blah" dest[108]="hdfs:///user/" +src[109]="es.xpack.user=hari" +dest[109]="es.xpack.user=" + +src[110]="es.xpack.password=myp@ss!" +dest[110]="es.xpack.password=" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From fa92d85f2ec120a3e493f9adbf82f14ec7d8568b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 16:07:09 +0000 Subject: [PATCH 0144/2779] extended username and password anonymization with generalization which also covers es.xpack.user= and es.xpack.password= --- anonymize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/anonymize.py b/anonymize.py index 1d698a8ee..e1a2cf387 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.1' +__version__ = '0.9.2' class Anonymize(CLI): @@ -295,7 +295,7 @@ def __init__(self): 'group2': r'({group_name}{sep}){user}'.format(group_name=group_name, sep=arg_sep, user=user_regex), 'group3': r'for\s+group\s+{group}'.format(group=user_regex), 'group4': r'(["\']{group_name}["\']\s*:\s*["\']?){group}'.format(group_name=group_name, group=user_regex), - 'user': r'(-{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), + 'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/(home|user)/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), 'user4': r'(?/){user}@'.format(user=user_regex), 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'\ .format(user_name=user_name, user=user_regex), - 'password': r'(-?{pass_word_phrase}{sep}){pw}'\ + 'password': r'([-\.]?{pass_word_phrase}{sep}){pw}'\ .format(pass_word_phrase=pass_word_phrase, sep=arg_sep, pw=password_quoted), From 0299d6927422cbbb92936694a95bfc46bdfae117 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 16:07:23 +0000 Subject: [PATCH 0145/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 56c74169c..c3fe41363 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 56c74169c5904b159505bbc38816dceb037daf57 +Subproject commit c3fe41363134922a0ab822084cb518579c8496ed From 2d96520e8d0f8c8be1dc897f5b00d109cbe96c07 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 17:00:48 +0000 Subject: [PATCH 0146/2779] updated anonymize.py --- anonymize.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/anonymize.py b/anonymize.py index e1a2cf387..4e5283dce 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,8 +90,10 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.2' +__version__ = '0.9.3' +ip_regex = '(?!127.0.0.1)' + ip_regex +subnet_mask_regex = '(?!127.0.0.1)' + subnet_mask_regex class Anonymize(CLI): @@ -274,7 +276,7 @@ def __init__(self): # don't use this simpler one as we want to catch everything inside quotes eg. 'my secret' #password_quoted = r'\S+' password_quoted = r'(?:\'[^\']+\'|"[^"]+"|\S+)' - user_name = r'(?:user(?:-?name)?|uid)' + user_name = r'(?:user(?:[_-]?name)?|uid)' group_name = r'group(?:-?name)?' arg_sep = r'[=\s:]+' # openssl uses -passin switch From 533ec06dc222bdeb544e41c3c8ba99725f760303 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 18:02:17 +0000 Subject: [PATCH 0147/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 3f2238f5e..ce87a6b69 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -424,6 +424,9 @@ dest[109]="es.xpack.user=" src[110]="es.xpack.password=myp@ss!" dest[110]="es.xpack.password=" +src[111]="127.0.0.1" +dest[111]="127.0.0.1" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 9535a9d6aad9138810390f62c5c845bb7070b8aa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 18:02:47 +0000 Subject: [PATCH 0148/2779] updated anonymize.py --- anonymize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/anonymize.py b/anonymize.py index 4e5283dce..e5fdc3b28 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,10 +90,10 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.3' +__version__ = '0.9.4' -ip_regex = '(?!127.0.0.1)' + ip_regex -subnet_mask_regex = '(?!127.0.0.1)' + subnet_mask_regex +ip_regex = r'(?!127\.0\.0\.)' + ip_regex +subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex class Anonymize(CLI): From a1dbbcd2abaedda03517b416926437ec60bad546 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 18:58:44 +0000 Subject: [PATCH 0149/2779] updated find_active_oozie.py --- find_active_oozie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/find_active_oozie.py b/find_active_oozie.py index 491afd818..21b9271d6 100755 --- a/find_active_oozie.py +++ b/find_active_oozie.py @@ -32,7 +32,7 @@ Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print NO_AVAILABLE_SERVER. -Tested on Oozie +Tested on Oozie 4.2.0 on Hortonworks HDP 2.6.0 """ From 042392d304ddf37ca0c7208c6188e654c08b91e6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 19:01:47 +0000 Subject: [PATCH 0150/2779] added find_active_kubernetes_api.py --- find_active_kubernetes_api.py | 86 +++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100755 find_active_kubernetes_api.py diff --git a/find_active_kubernetes_api.py b/find_active_kubernetes_api.py new file mode 100755 index 000000000..0ef8165f8 --- /dev/null +++ b/find_active_kubernetes_api.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Wed Sep 13 13:58:21 CEST 2017 +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to return the the active Kubernetes API server from an argument list of hosts + +Can mix and match between a comma separated list of hosts (--host server1,server2 or contents of the $HOST +environment variable if not specified) and general free-form space separated arguments, which is useful if piping +a host list through xargs. + +Multi-threaded for speed and exits upon first available host response to minimize delay to ~ 1 second or less. + +Useful for simplying scripting or generically extending tools that don't support HBase High Availability directly + +By default checks the same --port on all servers. Hosts may have optional : suffixes added to individually +override each one. + +Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print +NO_AVAILABLE_SERVER. + +Tested on Kubernetes 1.13 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +import traceback +#from random import shuffle +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from find_active_server import FindActiveServer +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.7.1' + + +class FindActiveKubernetesAPI(FindActiveServer): + + def __init__(self): + # Python 2.x + super(FindActiveKubernetesAPI, self).__init__() + # Python 3.x + # super().__init__() + self.default_port = 8001 + self.protocol = 'http' + # or just /healthz + self.url_path = '/healthz/ping' + self.regex = '^ok$' + self.default_num_threads = 2 + + def add_options(self): + self.add_hostoption(name='Kubernetes', default_port=self.default_port) + self.add_ssl_opt() + self.add_common_opts() + + def process_options(self): + self.validate_common_opts() + + +if __name__ == '__main__': + FindActiveKubernetesAPI().main() From f95626ec8ef4b94af02201158f58bd858328a8d7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Feb 2019 19:02:38 +0000 Subject: [PATCH 0151/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 54de3173f..ebe0c8ba1 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ Environment variables are supported for convenience and also to hide credentials - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node From 210170c133f0a7a451bddba8ac9419d827d77f96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 00:19:11 +0000 Subject: [PATCH 0152/2779] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index 83618d4cd..69c8ceca4 100755 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -13,10 +13,7 @@ # Deb Dev Package Requirements # ============================================================================ # -build-essential -libkrb5-dev libsasl2-dev -python-dev unzip wget zip From f336bcede38447ee35e72a1ffc4505d36b4c21c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 00:19:35 +0000 Subject: [PATCH 0153/2779] updated deb-packages-pip.txt --- setup/deb-packages-pip.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt index 9fe0d6594..0e36b47e1 100755 --- a/setup/deb-packages-pip.txt +++ b/setup/deb-packages-pip.txt @@ -27,6 +27,7 @@ python-linecache2 python-numpy python-psutil python-sh +python-six python-snappy python-thrift python-xmltodict From a165fb0af760d22a37cc24fd384ee2b3565c2594 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 00:20:06 +0000 Subject: [PATCH 0154/2779] updated deb-packages.txt --- setup/deb-packages.txt | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index c754b90a7..945f5e89d 100755 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -15,14 +15,11 @@ # needed to fetch the library submodule at end of build git - python -python-pip -python-setuptools -python-six -unzip + wget zip +unzip # needed for tests/test_spark* and ambari_cancel_all_requests.sh curl @@ -37,9 +34,6 @@ apache2-utils # needed for serf test's 'uptime' command procps -# needed to build python-snappy for avro module -libsnappy-dev - # installs 343MB of dependencies - install by hand if needed #ffmpeg From 1a59f95123d68d2bbc6f98e501826a184fe752c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 00:20:27 +0000 Subject: [PATCH 0155/2779] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index 69c8ceca4..b4728e2ec 100755 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -13,7 +13,12 @@ # Deb Dev Package Requirements # ============================================================================ # -libsasl2-dev +zip unzip wget -zip + +libsasl2-dev + +# needed to build python-snappy for avro module +libsnappy-dev + From 088e72182b04b5fc46ac6fc750fb64d25dd3cdcb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 00:20:34 +0000 Subject: [PATCH 0156/2779] updated deb-packages.txt --- setup/deb-packages.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index 945f5e89d..9630b31d7 100755 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -17,10 +17,6 @@ git python -wget -zip -unzip - # needed for tests/test_spark* and ambari_cancel_all_requests.sh curl From 0cd1f4af8757d2578a52a4bb8d1099a71d3de459 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 00:21:29 +0000 Subject: [PATCH 0157/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index dbc2428af..0ba7e6333 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit dbc2428af7dadbf15ccecb0178ded19e37bacb2e +Subproject commit 0ba7e6333940128b4b8205dd46e3eb0d620e8338 From 36ddba9875652fbb9a55fe0f0a4b811c9f59b53f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:54:00 +0000 Subject: [PATCH 0158/2779] updated brew-packages.txt --- setup/brew-packages.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index aa5eb8f28..cb53d439f 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -14,6 +14,4 @@ # Mac OS X - Homebrew Package Requirements # ============================================================================ # -openssl -wget yamllint From b95c96143f53372b2bd65ec93409741a09a32c9b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:55:12 +0000 Subject: [PATCH 0159/2779] updated rpm-packages-pip.txt --- setup/rpm-packages-pip.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt index 7e62dfc4d..72b514bcd 100755 --- a/setup/rpm-packages-pip.txt +++ b/setup/rpm-packages-pip.txt @@ -18,7 +18,6 @@ python-flask python-humanize python-jinja2 python-krbV -python-linecache2 python-markupsafe python-thrift python-xmltodict From 197f430206aa28946f99a27d12172a2f3e62e331 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:55:20 +0000 Subject: [PATCH 0160/2779] updated rpm-packages.txt --- setup/rpm-packages.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index fec3a42af..c55075910 100755 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -13,10 +13,6 @@ # RPM Package Requirements # ============================================================================ # -git -python-pip -python-setuptools - # needed for tests/test_spark* and ambari_cancel_all_requests.sh curl From ead457b3f4a5295562983625ec4c467414064418 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:55:24 +0000 Subject: [PATCH 0161/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index 83252f774..5edb73a58 100755 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -13,12 +13,9 @@ # RPM Dev Package Requirements # ============================================================================ # -cyrus-sasl-devel -gcc gcc-c++ -git + +cyrus-sasl-devel krb5-devel -python-devel unzip -wget zip From 4973af34a6e9a5fb5831b19d5bc9eae62a0f414d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:55:52 +0000 Subject: [PATCH 0162/2779] updated deb-packages.txt --- setup/deb-packages.txt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index c754b90a7..03fd9f06f 100755 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -13,13 +13,6 @@ # Deb Package Requirements # ============================================================================ # -# needed to fetch the library submodule at end of build -git - -python -python-pip -python-setuptools -python-six unzip wget zip From c448856b7cad39bbfd0703352fb5070268bf6f03 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:58:20 +0000 Subject: [PATCH 0163/2779] updated deb-packages.txt --- setup/deb-packages.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index 03fd9f06f..75b3309cd 100755 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -13,10 +13,6 @@ # Deb Package Requirements # ============================================================================ # -unzip -wget -zip - # needed for tests/test_spark* and ambari_cancel_all_requests.sh curl From f3c66641b9cee5759c30d8069f2fe7c942872351 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:58:28 +0000 Subject: [PATCH 0164/2779] updated deb-packages-pip.txt --- setup/deb-packages-pip.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt index 9fe0d6594..c970850bf 100755 --- a/setup/deb-packages-pip.txt +++ b/setup/deb-packages-pip.txt @@ -23,7 +23,6 @@ python-humanize python-jinja2 python-krbv python-ldif3 -python-linecache2 python-numpy python-psutil python-sh From e017eeb96e0b358fc3db3b3374d32c6ecc74a186 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:58:35 +0000 Subject: [PATCH 0165/2779] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index 83618d4cd..bfe169d52 100755 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -13,10 +13,5 @@ # Deb Dev Package Requirements # ============================================================================ # -build-essential -libkrb5-dev -libsasl2-dev -python-dev unzip -wget zip From 7394ee0bfe0380d1c306265cae36b30ccf56eb41 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:58:52 +0000 Subject: [PATCH 0166/2779] updated ../bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c3fe41363..5b0b37858 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c3fe41363134922a0ab822084cb518579c8496ed +Subproject commit 5b0b378583cb1a91365d91d132da8edf5606bb61 From 83aba085ef6359c9fccb311c5a477face807e7e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 10:58:52 +0000 Subject: [PATCH 0167/2779] updated ../pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 180c27175..803f396dd 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 180c271755adb13d96fe7a0a2703478e3e0a3429 +Subproject commit 803f396dd22f66eaf5c31905aa675438bcd71955 From a96c8be59eac8380865b5d00139b17c5413752a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 11:03:27 +0000 Subject: [PATCH 0168/2779] updated brew-packages.txt --- setup/brew-packages.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index cb53d439f..59edc4853 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -14,4 +14,5 @@ # Mac OS X - Homebrew Package Requirements # ============================================================================ # +snappy yamllint From d5e4f562180829bc2504d6332c16fe8c286879cf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 11:05:43 +0000 Subject: [PATCH 0169/2779] updated apk-packages-dev.txt --- setup/apk-packages-dev.txt | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt index 980b0ea40..01a5e07c6 100755 --- a/setup/apk-packages-dev.txt +++ b/setup/apk-packages-dev.txt @@ -12,15 +12,3 @@ # ============================================================================ # # Alpine Dev Package Requirements # ============================================================================ # - -alpine-sdk -gcc -cyrus-sasl-dev -krb5-dev -libffi-dev -linux-headers -#openssl-dev -libressl-dev -python-dev -wget -zip From 6688a82e10167d3a9a7259b53ed936d7f4dae757 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 11:07:05 +0000 Subject: [PATCH 0170/2779] updated apk-packages.txt --- setup/apk-packages.txt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt index a06d0f6ab..5b66956c2 100755 --- a/setup/apk-packages.txt +++ b/setup/apk-packages.txt @@ -13,15 +13,10 @@ # Alpine Package Requirements # ============================================================================ # -bash # without this arguments to mktemp fail in json tests coreutils -git + grep -make -py2-pip -python -snappy-dev # needed for tests/test_spark* and ambari_cancel_all_requests.sh curl From c180e31e82d297f4ca6c082c8b47892a54b3dc79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 11:07:11 +0000 Subject: [PATCH 0171/2779] updated apk-packages-dev.txt --- setup/apk-packages-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt index 01a5e07c6..c86db53a8 100755 --- a/setup/apk-packages-dev.txt +++ b/setup/apk-packages-dev.txt @@ -12,3 +12,5 @@ # ============================================================================ # # Alpine Dev Package Requirements # ============================================================================ # + +snappy-dev From c9396f5a3a9c8ad9541e67a49e6d1bf5d605eb1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 11:07:58 +0000 Subject: [PATCH 0172/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 5b0b37858..b714a0eb1 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 5b0b378583cb1a91365d91d132da8edf5606bb61 +Subproject commit b714a0eb19700f06c41ae4b5ebffb1f70b423b53 From 5bd2ae5398369704258571343e7bcfab64c08c00 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 11:07:58 +0000 Subject: [PATCH 0173/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 803f396dd..ad77e25b5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 803f396dd22f66eaf5c31905aa675438bcd71955 +Subproject commit ad77e25b5b322f60ec83b305acffb8ed7d7665e2 From 36745fa94ff8402e40667ec058eaaa64387b1f7d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 11:51:30 +0000 Subject: [PATCH 0174/2779] updated requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6e7ab2110..06b6c42db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,6 @@ humanize>=0.5.1 Jinja2>=2.8 #kazoo>=2.2.1 ldif3>=3.2.2 -linecache2>=1.0.0 MarkupSafe>=0.23 #Markdown>=2.6.8 numpy>=1.8.0rc1 From 5519b482e436724a5422fea63852f79b83328986 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 12:07:39 +0000 Subject: [PATCH 0175/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b714a0eb1..ad5679736 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b714a0eb19700f06c41ae4b5ebffb1f70b423b53 +Subproject commit ad56797369eaafa4514583bfc70050d9fae0e494 From e2840efe2c08c377223e6189ea01f5feae7e7bcd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 12:07:39 +0000 Subject: [PATCH 0176/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ad77e25b5..ea623c2d9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ad77e25b5b322f60ec83b305acffb8ed7d7665e2 +Subproject commit ea623c2d9fa91186e60a2cc4efe74db3adc58eec From a751bc465d3b501fa26c5af34320ed4713069cc0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 12:53:36 +0000 Subject: [PATCH 0177/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ad5679736..e516a4e34 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ad56797369eaafa4514583bfc70050d9fae0e494 +Subproject commit e516a4e346443f46a6a75604569b711ddc50bf32 From c3bd95c85d9de206ce76f4dd9bd1735b28080b66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 12:53:36 +0000 Subject: [PATCH 0178/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ea623c2d9..f3ceecca8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ea623c2d9fa91186e60a2cc4efe74db3adc58eec +Subproject commit f3ceecca84e7d2755d6d241ae5b9cdb48328d790 From d61f233db0d8e2b46d3685ebeb7fb0d6b1e9cb31 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Feb 2019 17:42:06 +0000 Subject: [PATCH 0179/2779] logged final running balance for a quick check against the PDF statement --- crunch_accounting_csv_statement_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index bbbd98f6d..7df68ca05 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5' +__version__ = '0.6.0' class CrunchAccountingCsvStatementConverter(CLI): @@ -107,6 +107,7 @@ def run(self): log.info("converted '%s' => '%s'", filename, target_filename) else: log.error("FAILED to convert filename '%s'", filename) + log.info('Final Balance: {}'.format(self.running_balance)) def convert(self, filename, target_filename): if self.reverse_order: From 72b4286cc529ac5bc5372beda3372ef28db3f6cf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Feb 2019 01:34:56 +0000 Subject: [PATCH 0180/2779] updated deb-packages-pip.txt --- setup/deb-packages-pip.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt index c970850bf..821f0a56f 100755 --- a/setup/deb-packages-pip.txt +++ b/setup/deb-packages-pip.txt @@ -14,9 +14,9 @@ # ============================================================================ # python-avro -python-bitarray +#python-bitarray python-dicttoxml -python-flask +#python-flask python-git python-happybase python-humanize From d9e15efb9ec3b7089699318ccb584240f1fb28b1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Feb 2019 01:35:09 +0000 Subject: [PATCH 0181/2779] updated rpm-packages-pip.txt --- setup/rpm-packages-pip.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt index 72b514bcd..5833b80dd 100755 --- a/setup/rpm-packages-pip.txt +++ b/setup/rpm-packages-pip.txt @@ -14,13 +14,13 @@ # ============================================================================ # GitPython -python-flask +#python-flask python-humanize python-jinja2 python-krbV -python-markupsafe +#python-markupsafe python-thrift python-xmltodict -python2-bitarray +#python2-bitarray python2-numpy python2-psutil From 164683c5227e15d96b352d78c1c8a772a5d2a0ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Feb 2019 01:35:30 +0000 Subject: [PATCH 0182/2779] updated requirements.txt --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 06b6c42db..0048cbfe2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ avro>=1.8.1 -bitarray>=0.8.1 +#bitarray>=0.8.1 #cassandra-driver>=3.6.0 dicttoxml>=1.7.4 # Elasticsearch library must match major version :( @@ -8,7 +8,7 @@ dicttoxml>=1.7.4 # for Elasticsearch 1.x #elasticsearch>=1.0.0,<2.0.0 # fails on requiring newer version of setuptools -Flask>=0.10.1 +#Flask>=0.10.1 GitPython>=2.0.8 happybase>=1.0.0 humanize>=0.5.1 @@ -16,7 +16,7 @@ humanize>=0.5.1 Jinja2>=2.8 #kazoo>=2.2.1 ldif3>=3.2.2 -MarkupSafe>=0.23 +#MarkupSafe>=0.23 #Markdown>=2.6.8 numpy>=1.8.0rc1 psutil>=4.3.0 From 253444135e8ec6cf64fc396c368a0f575c106146 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Feb 2019 01:36:11 +0000 Subject: [PATCH 0183/2779] updated apk-packages-pip.txt --- setup/apk-packages-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt index da9724ad7..a8f35241d 100755 --- a/setup/apk-packages-pip.txt +++ b/setup/apk-packages-pip.txt @@ -14,7 +14,7 @@ # ============================================================================ # py-dicttoxml -py-flask +#py-flask py-psutil py2-jinja2 py2-numpy From 80d3be934b6c8fa489460c105824d2303410d45b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Feb 2019 01:48:57 +0000 Subject: [PATCH 0184/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e516a4e34..4aaac4f7a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e516a4e346443f46a6a75604569b711ddc50bf32 +Subproject commit 4aaac4f7acd13b5f2d3cef79bf01c8c844a3ac36 From 8c1b85e5581f68ad79ada656f97bc9a378355281 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Feb 2019 01:48:57 +0000 Subject: [PATCH 0185/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f3ceecca8..536aa70b6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f3ceecca84e7d2755d6d241ae5b9cdb48328d790 +Subproject commit 536aa70b6fe203953be107591a7e6ffbf5c3da76 From 29a3e6970ba726cb6a044621771430dec15c1f4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Feb 2019 01:53:05 +0000 Subject: [PATCH 0186/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 536aa70b6..6a7811510 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 536aa70b6fe203953be107591a7e6ffbf5c3da76 +Subproject commit 6a7811510ee905a8cceb74f459477a1896486475 From 44dbd21e942e6ca5e3c038b5bc19cd5d9c35bd15 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Feb 2019 01:56:33 +0000 Subject: [PATCH 0187/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ebe0c8ba1..30423660b 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,8 @@ Environment variables are supported for convenience and also to hide credentials - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator + - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node From f40808ee43bb1301cedfdcc4102802e98d2f7f96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Feb 2019 02:04:02 +0000 Subject: [PATCH 0188/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4aaac4f7a..8fe158887 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4aaac4f7acd13b5f2d3cef79bf01c8c844a3ac36 +Subproject commit 8fe158887fbd13003fd758031a4a9e0964e2480f From d44d2a00bc3433dbe111bb2fcbdbd46b4f017897 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:25:02 +0000 Subject: [PATCH 0189/2779] updated brew-packages.txt --- setup/brew-packages.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index 59edc4853..49e653f65 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -14,5 +14,6 @@ # Mac OS X - Homebrew Package Requirements # ============================================================================ # +parallel snappy yamllint From b09359fb60cb8f17b24594638ab17f571f73afc4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:25:25 +0000 Subject: [PATCH 0190/2779] updated rpm-packages.txt --- setup/rpm-packages.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index c55075910..f32e8be3f 100755 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -26,6 +26,9 @@ https-tools # libgsasl-devel saslwrapper-devel #cyrus-sasl-devel +# for anonymize_parallel.sh +parallel + # needed to build python-snappy for avro module snappy-devel From 881edfda120056012fc082c0f69a13f791a18d95 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:25:32 +0000 Subject: [PATCH 0191/2779] updated brew-packages.txt --- setup/brew-packages.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index 49e653f65..f11d663a9 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -14,6 +14,8 @@ # Mac OS X - Homebrew Package Requirements # ============================================================================ # +# for anonymize_parallel.sh parallel + snappy yamllint From 6d892480a3e37f6d91c78ffc4e697357a548251d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:26:13 +0000 Subject: [PATCH 0192/2779] updated deb-packages.txt --- setup/deb-packages.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index 70210f5ab..7b7aaa0fc 100755 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -23,6 +23,9 @@ iputils-ping # for htpasswd for docker registry authenticated checks apache2-utils +# for anonymize_parallel.sh +parallel + # needed for serf test's 'uptime' command procps From e2cbc89d1cef95270506ef5073b5f6ee93458e53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:26:49 +0000 Subject: [PATCH 0193/2779] updated apk-packages.txt --- setup/apk-packages.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt index 5b66956c2..3e058fa61 100755 --- a/setup/apk-packages.txt +++ b/setup/apk-packages.txt @@ -24,6 +24,9 @@ curl # for htpasswd for docker registry authenticated checks apache2-utils +# for anonymize_parallel.sh +parallel + #which java || $(SUDO) apk add openjdk8-jre-base # Spark Java Py4J gets java linking error without this From 10e406dbce2118e5be4649e03f7e0c9e83df9d39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:29:24 +0000 Subject: [PATCH 0194/2779] updated apk-packages.txt --- setup/apk-packages.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt index 3e058fa61..9a9203026 100755 --- a/setup/apk-packages.txt +++ b/setup/apk-packages.txt @@ -25,6 +25,7 @@ curl apache2-utils # for anonymize_parallel.sh +split parallel #which java || $(SUDO) apk add openjdk8-jre-base From b7fd146c16adeb79e02dea02f762efcd9b5a363e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:35:40 +0000 Subject: [PATCH 0195/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8fe158887..3efa940b2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8fe158887fbd13003fd758031a4a9e0964e2480f +Subproject commit 3efa940b26f36061646820591c2ec07e691c6cec From 7857e4633a7c474085ea10b5cfe1ef53eb8e2456 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:37:32 +0000 Subject: [PATCH 0196/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3efa940b2..1b125b6c9 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3efa940b26f36061646820591c2ec07e691c6cec +Subproject commit 1b125b6c997a03b7df2c07258ee2a111ef73289c From a1ac76ce4063679a3508347f474352a1733b4e4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:45:49 +0000 Subject: [PATCH 0197/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b125b6c9..b628212ff 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b125b6c997a03b7df2c07258ee2a111ef73289c +Subproject commit b628212ff0c096a2ee454f4eacd954521f6e5beb From e502de47d324a6f6bd89538a037b8959370c3b9b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:49:50 +0000 Subject: [PATCH 0198/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b628212ff..05d0f3313 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b628212ff0c096a2ee454f4eacd954521f6e5beb +Subproject commit 05d0f33137f75b38b923ac136570eb4f5b346613 From 4dd2797b56aa5782d12dd3fe467e29a0e5bcce38 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:53:01 +0000 Subject: [PATCH 0199/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 05d0f3313..8797f500e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 05d0f33137f75b38b923ac136570eb4f5b346613 +Subproject commit 8797f500edfeec3cc0e1cc66869d4315356d13a9 From 4a1c67a84e4abd667165c91631c617c7a26e7ce4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 18:59:48 +0000 Subject: [PATCH 0200/2779] added anonymize_parallel.sh --- anonymize_parallel.sh | 63 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100755 anonymize_parallel.sh diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh new file mode 100755 index 000000000..ca4b86ed8 --- /dev/null +++ b/anonymize_parallel.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-03-05 18:27:00 +0000 (Tue, 05 Mar 2019) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +. "$srcdir/bash-tools/utils.sh" + +usage(){ + if [ -n "$*" ]; then + echo "$@" + echo + fi + cat < + +EOF + exit 3 +} + +for x in $@; do + case $x in + -h|--help) usage + ;; + esac +done + +parts="$(cpu_count)" + +for filename in $@; do + echo "Removing any pre-existing parts:" + rm -v "$filename".* + bash-tools/split.sh "$filename" + echo "Anonymizing parts" + for file_part in "$filename".*; do + echo "./anonymize.py -a $file_part > $file_part.anonymized" + done | + parallel -j "$parts" + cat "$filename".*.anonymized > "$filename".anonymized + echo "Removing parts:" + rm -v "$filename".*.anonymized + rm -v "$filename".[a-z0-9][a-z0-9] "$filename".[a-z0-9][a-z0-9][a-z0-9] 2>/dev/null + echo "Anonymized file ready: $filename.anonymized" +done +echo "Done" From ffdb5654867334f1dad4560d7722f46b6d7c844a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 20:22:55 +0000 Subject: [PATCH 0201/2779] updated apk-packages.txt --- setup/apk-packages.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt index 9a9203026..3e058fa61 100755 --- a/setup/apk-packages.txt +++ b/setup/apk-packages.txt @@ -25,7 +25,6 @@ curl apache2-utils # for anonymize_parallel.sh -split parallel #which java || $(SUDO) apk add openjdk8-jre-base From 987d5d4c06dae9c55b923e09b0821ec3adac8a46 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 20:23:31 +0000 Subject: [PATCH 0202/2779] updated apk-packages.txt --- setup/apk-packages.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt index 3e058fa61..66adf95d3 100755 --- a/setup/apk-packages.txt +++ b/setup/apk-packages.txt @@ -13,7 +13,8 @@ # Alpine Package Requirements # ============================================================================ # -# without this arguments to mktemp fail in json tests +# full mktemp needed for json tests +# full split needed for anonymize_parallel.sh coreutils grep From 14d0ce2b5cd0838a5f6c25eb940a2401e9972909 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 5 Mar 2019 20:26:26 +0000 Subject: [PATCH 0203/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 30423660b..46b6acd2a 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 30+ tools for Hadoop, NoSQL, Solr, Elasticsearch, Pig, Hive, Web URL + Nginx stats watchers, SQL and NoSQL syntax recasers, various Linux CLI tools +* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ tools for Hadoop, NoSQL, Solr, Elasticsearch, Pig, Hive, Web URL + Nginx stats watchers, SQL and NoSQL syntax recasers, various Linux CLI tools * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 400+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) From e84401171c7b732a561291f475910302beb86bb8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Mar 2019 10:17:20 +0000 Subject: [PATCH 0204/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8797f500e..6f75b8234 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8797f500edfeec3cc0e1cc66869d4315356d13a9 +Subproject commit 6f75b823432d1a006e38e24b33a07534340ed55d From 407c3bd0c65a9ed3dbd69cf935e7005e4ffd4c53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Mar 2019 10:17:20 +0000 Subject: [PATCH 0205/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6a7811510..c6bf7fa3f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6a7811510ee905a8cceb74f459477a1896486475 +Subproject commit c6bf7fa3f6ff0dd4913adaea32af848b92046131 From be6a39b7f787e1d4a0d092fdfdad9222e0043353 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Mar 2019 10:18:57 +0000 Subject: [PATCH 0206/2779] updated anonymize_parallel.sh --- anonymize_parallel.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index ca4b86ed8..c9625a3ae 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -19,6 +19,9 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . "$srcdir/bash-tools/utils.sh" +# re-establish srcdir local to this script since util.sh include brings its own srcdir +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + usage(){ if [ -n "$*" ]; then echo "$@" @@ -47,11 +50,11 @@ parts="$(cpu_count)" for filename in $@; do echo "Removing any pre-existing parts:" - rm -v "$filename".* - bash-tools/split.sh "$filename" + rm -v "$filename".* 2>/dev/null || : + "$srcdir/bash-tools/split.sh" "$filename" echo "Anonymizing parts" for file_part in "$filename".*; do - echo "./anonymize.py -a $file_part > $file_part.anonymized" + echo "$srcdir/anonymize.py -a $file_part > $file_part.anonymized" done | parallel -j "$parts" cat "$filename".*.anonymized > "$filename".anonymized From f62b46f0aba7c72f7a74ea45b19871d54d14ae77 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Mar 2019 11:23:37 +0000 Subject: [PATCH 0207/2779] updated anonymize_parallel.sh --- anonymize_parallel.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index c9625a3ae..3a8d4bd29 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -59,8 +59,8 @@ for filename in $@; do parallel -j "$parts" cat "$filename".*.anonymized > "$filename".anonymized echo "Removing parts:" - rm -v "$filename".*.anonymized - rm -v "$filename".[a-z0-9][a-z0-9] "$filename".[a-z0-9][a-z0-9][a-z0-9] 2>/dev/null + rm -v "$filename".*.anonymized || : + rm -v "$filename".[a-z0-9][a-z0-9] "$filename".[a-z0-9][a-z0-9][a-z0-9] 2>/dev/null || : echo "Anonymized file ready: $filename.anonymized" done echo "Done" From b7a16946f0c6bd9f1e896d8e625370ead2b1bc85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 9 Mar 2019 13:40:13 +0000 Subject: [PATCH 0208/2779] added option parsing --- anonymize_parallel.sh | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 3a8d4bd29..dca125142 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -29,34 +29,44 @@ usage(){ fi cat < +-p --parallelism Number of parts to split files in to and anonymize in paallel before reconstituting +-h --help Show usage and exit EOF exit 3 } -for x in $@; do - case $x in - -h|--help) usage - ;; +parallelism="${PARALLELISM:-$(cpu_count)}" + +file_list="" + +while [ $# -gt 0 ]; do + case $1 in + -p|--parallel) parallelism="$2" + shift + ;; + -h|--help|-*) usage + ;; + *) file_list="$file_list $1" + ;; esac + shift done -parts="$(cpu_count)" - -for filename in $@; do +for filename in $file_list; do echo "Removing any pre-existing parts:" rm -v "$filename".* 2>/dev/null || : - "$srcdir/bash-tools/split.sh" "$filename" + "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename" echo "Anonymizing parts" for file_part in "$filename".*; do echo "$srcdir/anonymize.py -a $file_part > $file_part.anonymized" done | - parallel -j "$parts" + parallel -j "$parallelism" cat "$filename".*.anonymized > "$filename".anonymized echo "Removing parts:" rm -v "$filename".*.anonymized || : From 6db6bd845392965298c30e7fd68d1250de7c9fd1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 9 Mar 2019 13:43:52 +0000 Subject: [PATCH 0209/2779] updated anonymize_parallel.sh --- anonymize_parallel.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index dca125142..0765c386f 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -59,18 +59,26 @@ while [ $# -gt 0 ]; do done for filename in $file_list; do + echo + echo "Processing file '$filename':" + echo echo "Removing any pre-existing parts:" rm -v "$filename".* 2>/dev/null || : + echo "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename" echo "Anonymizing parts" for file_part in "$filename".*; do - echo "$srcdir/anonymize.py -a $file_part > $file_part.anonymized" + cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized" + echo "$cmd" done | parallel -j "$parallelism" + echo "Concatenating parts" cat "$filename".*.anonymized > "$filename".anonymized + echo echo "Removing parts:" rm -v "$filename".*.anonymized || : rm -v "$filename".[a-z0-9][a-z0-9] "$filename".[a-z0-9][a-z0-9][a-z0-9] 2>/dev/null || : + echo echo "Anonymized file ready: $filename.anonymized" + echo done -echo "Done" From 7129d766de23152b61e78775626ae8d0fc846d1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 9 Mar 2019 13:56:15 +0000 Subject: [PATCH 0210/2779] updated Makefile --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 2898b573a..c0557195f 100755 --- a/Makefile +++ b/Makefile @@ -150,6 +150,9 @@ jython: if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi sh jython_install.sh +compile: + bash-tools/python_compile.sh + .PHONY: test-lib test-lib: cd pylib && $(MAKE) test From 1171ce6c1e9d4dc12ad1bc3a297ff56b70c943c6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 9 Mar 2019 13:57:45 +0000 Subject: [PATCH 0211/2779] updated Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c0557195f..3dc4960f0 100755 --- a/Makefile +++ b/Makefile @@ -150,7 +150,8 @@ jython: if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi sh jython_install.sh -compile: +.PHONY: pycompile +pycompile: bash-tools/python_compile.sh .PHONY: test-lib From d3080a8cb08fe6a16eac94206658223ed4e0af8e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 9 Mar 2019 13:57:59 +0000 Subject: [PATCH 0212/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3dc4960f0..b73c71272 100755 --- a/Makefile +++ b/Makefile @@ -98,7 +98,7 @@ python: # Python >= 2.7 - won't build on 2.6, handle separately and accept failure $(SUDO_PIP) pip install "ipython[notebook]" || : @echo - bash-tools/python_compile.sh + $(MAKE) pycompile @echo @echo $(MAKE) spark-deps From b49f1bb09009435d1758f396d75653580d153082 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 10 Mar 2019 11:54:35 +0000 Subject: [PATCH 0213/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 46b6acd2a..6503e33e9 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Environment variables are supported for convenience and also to hide credentials - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters + - ```anonymize_parallel.sh``` - splits files in to multiple parts to run anonymize.py on each part in parallel before re-join. Preserves order of evaluation important for anonymization rules, as well as file ordering. On servers this parallelization can result in a 30x speed up - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - Hadoop & NoSQL: From 44d2c34e18e0a0334c6254046d39a7f55534d664 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 10 Mar 2019 11:56:05 +0000 Subject: [PATCH 0214/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6503e33e9..7c3996944 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Environment variables are supported for convenience and also to hide credentials - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts to run anonymize.py on each part in parallel before re-join. Preserves order of evaluation important for anonymization rules, as well as file ordering. On servers this parallelization can result in a 30x speed up + - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as file ordering. On servers this parallelization can result in a 30x speed up - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - Hadoop & NoSQL: From f95b2e9b51217554563bcc424208b0204315e041 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 10 Mar 2019 11:56:48 +0000 Subject: [PATCH 0215/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c3996944..31395b164 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Environment variables are supported for convenience and also to hide credentials - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as file ordering. On servers this parallelization can result in a 30x speed up + - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining content order. On servers this parallelization can result in a 30x speed up - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - Hadoop & NoSQL: From 49e805cdf60cb504f6f201378d649b54fc694c95 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 10 Mar 2019 11:57:10 +0000 Subject: [PATCH 0216/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 31395b164..173d1b0ca 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Environment variables are supported for convenience and also to hide credentials - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining content order. On servers this parallelization can result in a 30x speed up + - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - Hadoop & NoSQL: From 26458e373b6b56ad215f7ee87be9c3165dd8750a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 10 Mar 2019 11:58:22 +0000 Subject: [PATCH 0217/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 173d1b0ca..8bc39884a 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Environment variables are supported for convenience and also to hide credentials - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up + - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - Hadoop & NoSQL: From 161965b6cc57c20de358f47052e28419acf968f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Mar 2019 10:25:16 +0000 Subject: [PATCH 0218/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c6bf7fa3f..75a7f43e6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c6bf7fa3f6ff0dd4913adaea32af848b92046131 +Subproject commit 75a7f43e6ba2a3f902af655fc548383a88826881 From f007a7b573fe6c4d35cdadf04d625547497c1d0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 14 Mar 2019 19:38:18 +0000 Subject: [PATCH 0219/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0048cbfe2..7aa194833 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ numpy>=1.8.0rc1 psutil>=4.3.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 -#python-ldap>=2.4.41 +python-ldap>=3.2.0 # PyLint fails some checks in Travis that pass locally #pyhs2>=0.6.0 #python-jenkins>=0.4.13 From 516dbacb2def5cf89e29d8035e73146af286c560 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 Mar 2019 17:05:48 +0000 Subject: [PATCH 0220/2779] Update anonymize_parallel.sh --- anonymize_parallel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 0765c386f..850e54737 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -29,7 +29,7 @@ usage(){ fi cat < Date: Fri, 15 Mar 2019 17:06:06 +0000 Subject: [PATCH 0221/2779] Update anonymize_parallel.sh --- anonymize_parallel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 850e54737..69c195ab4 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -29,7 +29,7 @@ usage(){ fi cat < Date: Fri, 15 Mar 2019 17:08:04 +0000 Subject: [PATCH 0222/2779] Update anonymize_parallel.sh --- anonymize_parallel.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 69c195ab4..eb04d41a9 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -24,10 +24,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" usage(){ if [ -n "$*" ]; then - echo "$@" - echo + echo "$@" >&2 + echo >&2 fi - cat <&2 < Date: Fri, 15 Mar 2019 17:09:41 +0000 Subject: [PATCH 0223/2779] Update anonymize_parallel.sh --- anonymize_parallel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index eb04d41a9..67d87e030 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -35,7 +35,7 @@ This makes it much, much faster to anonymize large log files for passing to vend usage: ${0##*/} --p --parallelism Number of parts to split files in to and anonymize in paallel before reconstituting +-p --parallelism Number of parts to split files in to and anonymize in parallel before reconstituting -h --help Show usage and exit EOF exit 3 From 1d125a4b4d766d02efafdb8f52466da996d1d83e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 12:11:35 +0000 Subject: [PATCH 0224/2779] updated deb-packages.txt --- setup/deb-packages.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index 7b7aaa0fc..e8cb7291c 100755 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -40,3 +40,5 @@ procps #ipython-notebook || : #which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk + +zip From 3e7ad716deaddca3b0de20d95f74f07abbeb7f09 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 12:11:42 +0000 Subject: [PATCH 0225/2779] updated apk-packages.txt --- setup/apk-packages.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt index 66adf95d3..fc6956249 100755 --- a/setup/apk-packages.txt +++ b/setup/apk-packages.txt @@ -32,3 +32,5 @@ parallel # Spark Java Py4J gets java linking error without this #if [ -f /lib/libc.musl-x86_64.so.1 ]; then [ -e /lib/ld-linux-x86-64.so.2 ] || ln -sv /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2; fi + +zip From 4d39512a4c95cb9c2d8cc856f11a6eaf49dc76a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 12:11:56 +0000 Subject: [PATCH 0226/2779] updated apk-packages-dev.txt --- setup/apk-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt index c86db53a8..8536eec0e 100755 --- a/setup/apk-packages-dev.txt +++ b/setup/apk-packages-dev.txt @@ -14,3 +14,4 @@ # ============================================================================ # snappy-dev +unzip From d6859fa6cee0dc79b31702df579c186a1132d204 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 12:12:07 +0000 Subject: [PATCH 0227/2779] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index b10916905..030a8296f 100755 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -15,3 +15,4 @@ # needed to build python-snappy for avro module libsnappy-dev +unzip From dffb28315e6cc22768493b8e5069b2cf3bdea541 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 12:57:51 +0000 Subject: [PATCH 0228/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6f75b8234..64f6ed2fd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6f75b823432d1a006e38e24b33a07534340ed55d +Subproject commit 64f6ed2fdee83f983b89d672e079833bcb2b7d15 From b49166baf181d6c32fb592ee82483d6b825d5f01 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 12:57:51 +0000 Subject: [PATCH 0229/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 75a7f43e6..5ebae8839 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 75a7f43e6ba2a3f902af655fc548383a88826881 +Subproject commit 5ebae88398c160e61cb186c672b7d8cec701a016 From 58cb75bef633b592409b7e2cf94b6cce5cede6d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 14:49:22 +0000 Subject: [PATCH 0230/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index 5edb73a58..0c63dc7d5 100755 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -14,8 +14,9 @@ # ============================================================================ # gcc-c++ - cyrus-sasl-devel krb5-devel +openssl-devel +snappy-devel unzip zip From c7f084050ac4aae665bbfe2b1bbfb8e0969a133a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 14:55:02 +0000 Subject: [PATCH 0231/2779] updated rpm-packages-pip.txt --- setup/rpm-packages-pip.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt index 5833b80dd..db112cbc7 100755 --- a/setup/rpm-packages-pip.txt +++ b/setup/rpm-packages-pip.txt @@ -14,13 +14,14 @@ # ============================================================================ # GitPython -#python-flask python-humanize python-jinja2 python-krbV -#python-markupsafe +python-ldap python-thrift python-xmltodict -#python2-bitarray python2-numpy python2-psutil +#python-flask +#python-markupsafe +#python2-bitarray From 4cd2a05955c167b4c1b3cd1da9fe3f6ab2bd5828 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 14:55:47 +0000 Subject: [PATCH 0232/2779] updated deb-packages-pip.txt --- setup/deb-packages-pip.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt index 821f0a56f..75196cfb9 100755 --- a/setup/deb-packages-pip.txt +++ b/setup/deb-packages-pip.txt @@ -14,14 +14,13 @@ # ============================================================================ # python-avro -#python-bitarray python-dicttoxml -#python-flask python-git python-happybase python-humanize python-jinja2 python-krbv +python-ldap python-ldif3 python-numpy python-psutil @@ -29,3 +28,5 @@ python-sh python-snappy python-thrift python-xmltodict +#python-bitarray +#python-flask From 212732a3448d992c78658ecbcc52c7fdafde984e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 14:56:53 +0000 Subject: [PATCH 0233/2779] updated apk-packages-pip.txt --- setup/apk-packages-pip.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt index a8f35241d..88bcf4c3a 100755 --- a/setup/apk-packages-pip.txt +++ b/setup/apk-packages-pip.txt @@ -14,7 +14,8 @@ # ============================================================================ # py-dicttoxml -#py-flask py-psutil +py-pyldap py2-jinja2 py2-numpy +#py-flask From e24cbfeaeada9fa87e0aab6864bf6e8784e597e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 14:57:56 +0000 Subject: [PATCH 0234/2779] updated setup --- setup/apk-packages-dev.txt | 0 setup/apk-packages-pip.txt | 0 setup/apk-packages.txt | 0 setup/deb-packages-dev.txt | 0 setup/deb-packages-optional.txt | 0 setup/deb-packages-pip.txt | 0 setup/deb-packages.txt | 0 setup/rpm-packages-dev.txt | 0 setup/rpm-packages-optional.txt | 0 setup/rpm-packages-pip.txt | 0 setup/rpm-packages.txt | 0 11 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 setup/apk-packages-dev.txt mode change 100755 => 100644 setup/apk-packages-pip.txt mode change 100755 => 100644 setup/apk-packages.txt mode change 100755 => 100644 setup/deb-packages-dev.txt mode change 100755 => 100644 setup/deb-packages-optional.txt mode change 100755 => 100644 setup/deb-packages-pip.txt mode change 100755 => 100644 setup/deb-packages.txt mode change 100755 => 100644 setup/rpm-packages-dev.txt mode change 100755 => 100644 setup/rpm-packages-optional.txt mode change 100755 => 100644 setup/rpm-packages-pip.txt mode change 100755 => 100644 setup/rpm-packages.txt diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt old mode 100755 new mode 100644 diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt old mode 100755 new mode 100644 diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt old mode 100755 new mode 100644 diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt old mode 100755 new mode 100644 diff --git a/setup/deb-packages-optional.txt b/setup/deb-packages-optional.txt old mode 100755 new mode 100644 diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt old mode 100755 new mode 100644 diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt old mode 100755 new mode 100644 diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt old mode 100755 new mode 100644 diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt old mode 100755 new mode 100644 diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt old mode 100755 new mode 100644 diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt old mode 100755 new mode 100644 From ee5647c7ec98a1824a56dec5a989343470e14f1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 14:59:12 +0000 Subject: [PATCH 0235/2779] updated apk-packages-dev.txt --- setup/apk-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt index 8536eec0e..f8738775b 100644 --- a/setup/apk-packages-dev.txt +++ b/setup/apk-packages-dev.txt @@ -13,5 +13,6 @@ # Alpine Dev Package Requirements # ============================================================================ # +openldap-dev snappy-dev unzip From a0b27e0a52619b338e85dcbf19ea0c49e8af1e2b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 14:59:59 +0000 Subject: [PATCH 0236/2779] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index 030a8296f..86dd75dc0 100644 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -13,6 +13,7 @@ # Deb Dev Package Requirements # ============================================================================ # +libldap2-dev # needed to build python-snappy for avro module libsnappy-dev unzip From 27b9d1063eeebcd4810e0302aa30952cd991d3ca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 15:05:43 +0000 Subject: [PATCH 0237/2779] updated rpm-packages.txt --- setup/rpm-packages.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index f32e8be3f..4317c6416 100644 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -29,7 +29,4 @@ https-tools # for anonymize_parallel.sh parallel -# needed to build python-snappy for avro module -snappy-devel - java From e8f0b0255a98e9ba9798a5c0831da8a71ed63510 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 15:05:55 +0000 Subject: [PATCH 0238/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index 0c63dc7d5..e78705f18 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -17,6 +17,7 @@ gcc-c++ cyrus-sasl-devel krb5-devel openssl-devel +# needed to build python-snappy for avro module snappy-devel unzip zip From 6a3abfefea4b11f75667efa3beeb67999852d9b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 22:52:24 +0000 Subject: [PATCH 0239/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 64f6ed2fd..1f8b62e98 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 64f6ed2fdee83f983b89d672e079833bcb2b7d15 +Subproject commit 1f8b62e984f71a3d034d6aacba6876967cf847b6 From 42ee8592af1db2dd1addb0457284893f24261b57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 Mar 2019 22:52:24 +0000 Subject: [PATCH 0240/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5ebae8839..8601b21dd 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5ebae88398c160e61cb186c672b7d8cec701a016 +Subproject commit 8601b21dd46ea68b0a042fcce8964762346f6c99 From ebcb92b5d35050dc24fcc901ffe4c1067c16641b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Mar 2019 20:34:39 +0000 Subject: [PATCH 0241/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index e78705f18..c15564a57 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -19,5 +19,3 @@ krb5-devel openssl-devel # needed to build python-snappy for avro module snappy-devel -unzip -zip From 76066ee6b31d4df5f842e9ad042202afdea79e42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Mar 2019 20:35:21 +0000 Subject: [PATCH 0242/2779] updated rpm-packages.txt --- setup/rpm-packages.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index 4317c6416..281cfb022 100644 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -13,6 +13,8 @@ # RPM Package Requirements # ============================================================================ # +java + # needed for tests/test_spark* and ambari_cancel_all_requests.sh curl @@ -29,4 +31,5 @@ https-tools # for anonymize_parallel.sh parallel -java +unzip +zip From 11f5d0fc96f52545a330803c0456f14826a44307 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Mar 2019 20:35:27 +0000 Subject: [PATCH 0243/2779] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index 86dd75dc0..e6936942c 100644 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -16,4 +16,3 @@ libldap2-dev # needed to build python-snappy for avro module libsnappy-dev -unzip From fce499b03ae3b100552eb17de4d78c8fcdb14d21 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 17 Mar 2019 20:35:32 +0000 Subject: [PATCH 0244/2779] updated deb-packages.txt --- setup/deb-packages.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index e8cb7291c..c605fdd27 100644 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -42,3 +42,4 @@ procps #which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk zip +unzip From 6ecc609e9e5c2f496a73459d628c12b556bde6d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Mar 2019 17:52:32 +0000 Subject: [PATCH 0245/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1f8b62e98..e5ac7d9bd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1f8b62e984f71a3d034d6aacba6876967cf847b6 +Subproject commit e5ac7d9bdcf433583e5a10c6bf23e7df3e9ecebb From bd809c9488d5653994c7d0f554ebed4f55005d4b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Mar 2019 17:52:32 +0000 Subject: [PATCH 0246/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8601b21dd..70dbba607 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8601b21dd46ea68b0a042fcce8964762346f6c99 +Subproject commit 70dbba607b0abf522ed963786fffdfe169495ec9 From 8d1645e5f8dfdafa050a9132962a7f85a02a5e93 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Mar 2019 18:09:09 +0000 Subject: [PATCH 0247/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e5ac7d9bd..1b02ade81 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e5ac7d9bdcf433583e5a10c6bf23e7df3e9ecebb +Subproject commit 1b02ade8112a825103a88487ddc4184c2f951247 From 6d285b8905ace8472a67c8c5582a8937a2952dd8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Mar 2019 18:09:10 +0000 Subject: [PATCH 0248/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 70dbba607..d0e42e4f6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 70dbba607b0abf522ed963786fffdfe169495ec9 +Subproject commit d0e42e4f6a2fe4d5af7c4167d51027b81b6858be From f2d08b8396ff848313c1a145b4785b7abaebedd0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Mar 2019 19:17:09 +0000 Subject: [PATCH 0249/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1f8b62e98..1b02ade81 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1f8b62e984f71a3d034d6aacba6876967cf847b6 +Subproject commit 1b02ade8112a825103a88487ddc4184c2f951247 From 74bee865b0c1ad7aac35411b6ea8d94fee97a871 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Mar 2019 19:17:09 +0000 Subject: [PATCH 0250/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8601b21dd..0d0dff9d9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8601b21dd46ea68b0a042fcce8964762346f6c99 +Subproject commit 0d0dff9d98ff953e096d23d30b5d14ca30d50271 From 96a3ea4b82d94702f648df674e04e78934fcc1ef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Mar 2019 11:47:59 +0000 Subject: [PATCH 0251/2779] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8bc39884a..2aee6c04f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ Hari Sekhon - DevOps Python Tools ================================= -[![Build Status](https://travis-ci.org/HariSekhon/devops-python-tools.svg?branch=master)](https://travis-ci.org/HariSekhon/devops-python-tools) +[![Build Status](https://travis-ci.org/HariSekhon/DevOps-Python-tools.svg?branch=master)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) From 49cc4051ac25f2f4b2fffa5e99b72eab12f16f68 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 Mar 2019 22:20:30 +0000 Subject: [PATCH 0252/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0d0dff9d9..1fca7bd56 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0d0dff9d98ff953e096d23d30b5d14ca30d50271 +Subproject commit 1fca7bd56bfbdfd5adf21bd384620c9215e0b4e4 From d75ab524354720326b82d411139197c66e41a678 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Mar 2019 23:25:40 +0000 Subject: [PATCH 0253/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 7df68ca05..8b3f6e80e 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.0' +__version__ = '0.6.1' class CrunchAccountingCsvStatementConverter(CLI): @@ -164,7 +164,8 @@ def detect_columns(self, csvreader): positions = {'date': None, 'desc': None, 'amount': None} balance_position = None for (position, value) in enumerate(headers): - if 'Date' in value: + # want Transaction Date and not Posted Date + if 'Date' in value and not 'Posted' in value: positions['date'] = position elif 'Merchant Name' in value: positions['desc'] = position From 9eacd0581ec0608e4cd9038eebd3b8ed1cebec14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 12 Apr 2019 18:01:52 +0100 Subject: [PATCH 0254/2779] added ldap_download.sh, ldap_upload.sh --- tests/data/ldap_download.sh | 20 ++++++++++++++++++++ tests/data/ldap_upload.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100755 tests/data/ldap_download.sh create mode 100755 tests/data/ldap_upload.sh diff --git a/tests/data/ldap_download.sh b/tests/data/ldap_download.sh new file mode 100755 index 000000000..1eb3dcd6f --- /dev/null +++ b/tests/data/ldap_download.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +ldapsearch -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -b dc=example,dc=org > "$srcdir/ldap.ldif" diff --git a/tests/data/ldap_upload.sh b/tests/data/ldap_upload.sh new file mode 100755 index 000000000..0ff49ca02 --- /dev/null +++ b/tests/data/ldap_upload.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# -a add +# -c continue on error (if already exists) + +# Insufficient Permissions :-/ +#ldapmodify -a -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap_memberof_overlay.ldif" + +# add +#ldapmodify -a -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap.ldif" || : +ldapadd -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap.ldif" + +# modify +#ldapmodify -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap.ldif" From 2492df8e51a6ac5934d624399434a616e4907f26 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 12 Apr 2019 18:02:05 +0100 Subject: [PATCH 0255/2779] added ldap.ldif --- tests/data/ldap.ldif | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/data/ldap.ldif diff --git a/tests/data/ldap.ldif b/tests/data/ldap.ldif new file mode 100644 index 000000000..dbffa7fed --- /dev/null +++ b/tests/data/ldap.ldif @@ -0,0 +1,70 @@ +# extended LDIF +# +# LDAPv3 +# base with scope subtree +# filter: (objectclass=*) +# requesting: ALL +# + +# example.org +dn: dc=example,dc=org +objectClass: top +objectClass: dcObject +objectClass: organization +o: Example Inc. +dc: example + +# admin, example.org +dn: cn=admin,dc=example,dc=org +objectClass: simpleSecurityObject +objectClass: organizationalRole +cn: admin +description: LDAP administrator +userPassword:: e1NTSEF9TnpTTnRqbzZUUlgrM2c3Y3c4VHU3eVpCOXhaaDJtN00= + +# Users, example.org +dn: ou=Users,dc=example,dc=org +objectClass: organizationalUnit +objectClass: top +ou: Users + +# Groups, example.org +dn: ou=Groups,dc=example,dc=org +objectClass: organizationalUnit +objectClass: top +ou: Groups + +# Dept1, Groups, example.org +dn: ou=Dept1,ou=Groups,dc=example,dc=org +objectClass: organizationalUnit +objectClass: top +ou: Dept1 + +# DeptA, Dept1, Groups, example.org +dn: ou=DeptA,ou=Dept1,ou=Groups,dc=example,dc=org +objectClass: organizationalUnit +objectClass: top +ou: DeptA + +# hari, Users, example.org +dn: cn=hari,ou=Users,dc=example,dc=org +cn: hari +objectClass: organizationalPerson +objectClass: person +objectClass: top +sn: sekhon +memberOf: cn=SuperAdmins,ou=DeptA,ou=Dept1,ou=Groups,dc=example,dc=org + +# SuperAdmins, DeptA, Dept1, Groups, example.org +dn: cn=SuperAdmins,ou=DeptA,ou=Dept1,ou=Groups,dc=example,dc=org +cn: SuperAdmins +member: cn=hari,ou=Users,dc=example,dc=org +objectClass: groupOfNames +objectClass: top + +# search result +search: 2 +result: 0 Success + +# numResponses: 9 +# numEntries: 8 From da13c2de70cb122f22e83c92e9601752125d0fb8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 14 Apr 2019 12:51:37 +0100 Subject: [PATCH 0256/2779] updated ldap.ldif --- tests/data/ldap.ldif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/ldap.ldif b/tests/data/ldap.ldif index dbffa7fed..b9786d768 100644 --- a/tests/data/ldap.ldif +++ b/tests/data/ldap.ldif @@ -63,8 +63,8 @@ objectClass: groupOfNames objectClass: top # search result -search: 2 -result: 0 Success +#search: 2 +#result: 0 Success # numResponses: 9 # numEntries: 8 From f3860a2e75b1fefff411619f9d46a94eead83b4e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 May 2019 19:56:14 +0100 Subject: [PATCH 0257/2779] updated tests/test_validate_yaml.sh --- tests/test_validate_yaml.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_validate_yaml.sh b/tests/test_validate_yaml.sh index c2efb60a0..b0391525b 100755 --- a/tests/test_validate_yaml.sh +++ b/tests/test_validate_yaml.sh @@ -96,8 +96,7 @@ check_broken(){ cat > "$broken_dir/broken_tabs.yaml" < Date: Thu, 2 May 2019 19:56:56 +0100 Subject: [PATCH 0258/2779] switched to yaml.safe_load() --- validate_yaml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_yaml.py b/validate_yaml.py index 42b8bf13c..2852a1905 100755 --- a/validate_yaml.py +++ b/validate_yaml.py @@ -49,7 +49,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.0' +__version__ = '0.9.1' class YamlValidatorTool(CLI): @@ -99,7 +99,7 @@ def check_yaml(self, content): if not self.get_opt('print'): if self.verbose > 2: try: - yaml.load(content) + yaml.safe_load(content) except yaml.YAMLError as _: print(_) die(self.invalid_yaml_msg) From 404f1dca5b5868dbd9d97bbee387bdcf81f91314 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 May 2019 20:05:24 +0100 Subject: [PATCH 0259/2779] updated hbase_table_regions_by_regionserver.sh --- hbase_table_regions_by_regionserver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase_table_regions_by_regionserver.sh b/hbase_table_regions_by_regionserver.sh index b19510d8f..6ba8244ff 100755 --- a/hbase_table_regions_by_regionserver.sh +++ b/hbase_table_regions_by_regionserver.sh @@ -40,7 +40,7 @@ usage: ${0##*/} [options] regionserver1 regionserver2 regionserver3 ... -p --port RegionServer HTTP JMX port EOF - exit 1 + exit 3 } table=".*" From db4b093776204132c8e85f1c28e5878d6b29a7c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 May 2019 20:10:04 +0100 Subject: [PATCH 0260/2779] added --skip-zeros --- hbase_region_requests.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/hbase_region_requests.py b/hbase_region_requests.py index 68ec05500..9c5586c7c 100755 --- a/hbase_region_requests.py +++ b/hbase_region_requests.py @@ -50,7 +50,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.2' +__version__ = '0.5.3' class HBaseRegionsRequests(CLI): @@ -88,6 +88,7 @@ def add_options(self): self.add_opt('--reads', action='store_true', help='Show read requests (default shows read and write)') self.add_opt('--writes', action='store_true', help='Show write requests (default shows read and write') self.add_opt('--total', action='store_true', help='Show total requests (default shows read and write)') + self.add_opt('--skip-zeros', action='store_true', help="Don't output regions which have zero requests") def process_args(self): self.host_list = self.args @@ -211,6 +212,7 @@ def process_bean(self, host, bean, uptime): def print_stats(self, host): stats = self.stats show = self.show + skip_zeros = self.get_opt('skip_zeros') tstamp = time.strftime('%F %T') if not stats: print("No table regions found for table '{}'. Did you specify the correct table name?".format(self.table)) @@ -221,6 +223,7 @@ def print_stats(self, host): .format(tstamp, host, self.interval, plural(self.interval))) self.first_iteration = 0 return + count = 0 for table in sorted(stats[host]): for region in sorted(stats[host][table]): table_region = region @@ -229,10 +232,16 @@ def print_stats(self, host): # maintain explicit order for humans # rather than iterate keys of region which will some out in the wrong order for metric in ('read', 'write', 'total'): - if (not show) or metric in show: - print('{:20s}\t{:20s}\t{:40s}\t{:10s}\t{:8.0f}'\ - .format(tstamp, host, table_region, metric, stats[host][table][region][metric])) - print() + value = stats[host][table][region][metric] + if value == 0 and skip_zeros: + continue + if show and metric not in show: + continue + print('{:20s}\t{:20s}\t{:40s}\t{:10s}\t{:8.0f}'\ + .format(tstamp, host, table_region, metric, value)) + count += 1 + if count: + print() # some extra effort to make it look the same as HBase presents it as #def encode_char(self, char): From de62b683fc18157855d73a892e76572e1bf34ec6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 May 2019 20:10:17 +0100 Subject: [PATCH 0261/2779] updated tests/test_hbase.sh --- tests/test_hbase.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_hbase.sh b/tests/test_hbase.sh index b107bc7e3..aaa315dc3 100755 --- a/tests/test_hbase.sh +++ b/tests/test_hbase.sh @@ -247,8 +247,10 @@ EOF # ============================================================================ # run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 1 run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 1 --average + run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 1 --average --skip-zeros run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 1 + run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 1 --skip-zeros run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 1 --average run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST --count 2 --interval 2 From 7ae0ddb64f0d2341987a965e91bf6c936e4e0b1d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 May 2019 20:34:03 +0100 Subject: [PATCH 0262/2779] updated tests/test_hbase.sh --- tests/test_hbase.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_hbase.sh b/tests/test_hbase.sh index aaa315dc3..fdcc7d7b6 100755 --- a/tests/test_hbase.sh +++ b/tests/test_hbase.sh @@ -245,19 +245,19 @@ EOF run_conn_refused ./hbase_table_row_key_distribution.py -T HexStringSplitTable # ============================================================================ # - run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 1 - run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 1 --average - run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 1 --average --skip-zeros + run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 + run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 --average + run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 --average --skip-zeros - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 1 - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 1 --skip-zeros - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 1 --average + run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 + run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 --skip-zeros + run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 --average run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST --count 2 --interval 2 - run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 1 + run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 2 run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST --count 2 -i 2 - run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 1 --average + run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 2 --average # ============================================================================ # run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 From 3d46936863bc81ea19d81317532d22239b639068 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 May 2019 20:49:17 +0100 Subject: [PATCH 0263/2779] updated hbase_region_requests.py --- hbase_region_requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase_region_requests.py b/hbase_region_requests.py index 9c5586c7c..2af24c219 100755 --- a/hbase_region_requests.py +++ b/hbase_region_requests.py @@ -233,7 +233,7 @@ def print_stats(self, host): # rather than iterate keys of region which will some out in the wrong order for metric in ('read', 'write', 'total'): value = stats[host][table][region][metric] - if value == 0 and skip_zeros: + if skip_zeros and int(value) == 0: continue if show and metric not in show: continue From bd92c7c5dc5831a31424fb6400d89d336d606d8c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 May 2019 20:44:52 +0100 Subject: [PATCH 0264/2779] cast amount to Decimal in amount method --- crunch_accounting_csv_statement_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 8b3f6e80e..3c0ef9a3e 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.1' +__version__ = '0.6.2' class CrunchAccountingCsvStatementConverter(CLI): @@ -144,7 +144,7 @@ def convert(self, filename, target_filename): def amount(self, amount): if self.credit_card: return -Decimal(amount) - return amount + return Decimal(amount) @staticmethod def reverse_contents(filename): From cc3337db2a29ba3847ceb9fb1dad435cf5f57ba1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 17 Jul 2019 17:11:25 +0100 Subject: [PATCH 0265/2779] set dist trusty --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index fcffb06e8..9d5678f9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,8 @@ python: # - "pypy" # - "pypy3" +dist: trusty + sudo: required env: From ef1a7ee49f47bbc56592708b964f99826500bff6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 Jul 2019 12:18:02 +0100 Subject: [PATCH 0266/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2aee6c04f..309b2de93 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Environment variables are supported for convenience and also to hide credentials - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) -- Hadoop & NoSQL: +- [Hadoop](http://hadoop.apache.org/) & NoSQL: - [Spark](https://spark.apache.org/) & Data Format Converters: - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter @@ -141,7 +141,7 @@ Environment variables are supported for convenience and also to hide credentials - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (official `docker search` is limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` -- Travis CI: +- [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI - Data Validation (useful in CI): From 346b33b9993f59b5a1ba66b74217ed5b6c180b67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 Jul 2019 13:41:12 +0100 Subject: [PATCH 0267/2779] updated travis_last_log.py --- travis_last_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis_last_log.py b/travis_last_log.py index e7a20992b..54f3ccc5f 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -33,7 +33,7 @@ If repo starts with / and $TRAVIS_USER is set, will prepend $TRAVIS_USER for convenience, eg. export TRAVIS_USER=HariSekhon # put this in .bashrc -travis_last_log.py /pytools # shorter to type '/pytools' than 'HariSekhon/pytools' +travis_last_log.py /DevOps-Python-tools # shorter to type than full 'HariSekhon/DevOps-Python-tools' As a convenience you may supply either job id or repo as an argument without any switch and it'll infer it as a repo if if contains a slash but no url (eg. HariSekhon/nagios-plugins) otherwise it'll assume it's a job id, strip any leading @@ -50,7 +50,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -#from __future__ import unicode_literals +from __future__ import unicode_literals import json import logging From 2cfab91f1e177d9e32171228c1859ffebf60de85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 Jul 2019 13:41:17 +0100 Subject: [PATCH 0268/2779] updated .yamllint --- .yamllint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamllint b/.yamllint index d3ed7f7ac..b5e6f8448 100644 --- a/.yamllint +++ b/.yamllint @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019) # -# https://github.com/harisekhon/pytools +# https://github.com/harisekhon/devops-python-tools # # License: see accompanying Hari Sekhon LICENSE file # From 00fa384fa9da7f2b42db80d4c61f260f1858bab9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 Jul 2019 13:41:57 +0100 Subject: [PATCH 0269/2779] updated hdfs_find_replication_factor_1.py --- hdfs_find_replication_factor_1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py index 516122e7b..fa80c3586 100755 --- a/hdfs_find_replication_factor_1.py +++ b/hdfs_find_replication_factor_1.py @@ -5,7 +5,7 @@ # Author: Hari Sekhon # Date: 2018-11-28 16:37:00 +0000 (Wed, 28 Nov 2018) # -# https://github.com/harisekhon/pytools +# https://github.com/harisekhon/devops-python-tools # # License: see accompanying Hari Sekhon LICENSE file # From 757218785c251f3812a055ee5b1fac42ac62357a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 Jul 2019 13:42:33 +0100 Subject: [PATCH 0270/2779] updated anonymize_parallel.sh --- anonymize_parallel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 67d87e030..b12071d6b 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -4,7 +4,7 @@ # Author: Hari Sekhon # Date: 2019-03-05 18:27:00 +0000 (Tue, 05 Mar 2019) # -# https://github.com/harisekhon/pytools +# https://github.com/harisekhon/devops-python-tools # # License: see accompanying Hari Sekhon LICENSE file # From 2a01a5b8961799ca3b46db8d3466476adea7f0f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 Jul 2019 13:42:49 +0100 Subject: [PATCH 0271/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 3c0ef9a3e..34069c12b 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -5,7 +5,7 @@ # Author: Hari Sekhon # Date: 2019-02-04 23:24:30 +0000 (Mon, 04 Feb 2019) # -# https://github.com/harisekhon/pytools +# https://github.com/harisekhon/devops-python-tools # # License: see accompanying Hari Sekhon LICENSE file # From f6fbc2d74a7a1f3bd5b3681b02d295ad9beb5bd1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 30 Jul 2019 18:35:05 +0100 Subject: [PATCH 0272/2779] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 309b2de93..1e08ff3d3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ Hari Sekhon - DevOps Python Tools [![Codacy Badge](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) -[![Dependency Status](https://gemnasium.com/badges/github.com/HariSekhon/devops-python-tools.svg)](https://gemnasium.com/github.com/HariSekhon/devops-python-tools) [![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) From 2cedc9bad174134d008c9a31e650c7f9e8f637f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 31 Jul 2019 19:05:38 +0100 Subject: [PATCH 0273/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b02ade81..9a54f2060 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b02ade8112a825103a88487ddc4184c2f951247 +Subproject commit 9a54f20609c45c0624120866ef2eb309a60acb9e From bca6f5dc9daef3495100ac4de381f215dad28fa9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 31 Jul 2019 19:06:01 +0100 Subject: [PATCH 0274/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1fca7bd56..5248361e5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1fca7bd56bfbdfd5adf21bd384620c9215e0b4e4 +Subproject commit 5248361e58ce15b0c2465f2c52a3ba6e31f951f9 From 79df678440bf72ad96cf2a171f25d49343fd929f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 31 Jul 2019 19:08:31 +0100 Subject: [PATCH 0275/2779] updated bash tools utils references --- anonymize_parallel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index b12071d6b..5b8a1fb4c 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -17,7 +17,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -. "$srcdir/bash-tools/utils.sh" +. "$srcdir/bash-tools/lib/utils.sh" # re-establish srcdir local to this script since util.sh include brings its own srcdir srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" From 3267071f57eba3dd3e2c17a78a74b29e19b5b1ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 31 Jul 2019 19:08:36 +0100 Subject: [PATCH 0276/2779] updated bash tools utils references --- tests/all.sh | 2 +- tests/test_center.sh | 2 +- tests/test_docker.sh | 2 +- tests/test_docker_registry_show_tags.sh | 2 +- tests/test_dockerfiles_check_git_branches.sh | 2 +- tests/test_dockerfiles_check_git_tags.sh | 2 +- tests/test_dockerhub_search.sh | 2 +- tests/test_dockerhub_show_tags.sh | 2 +- tests/test_find_active_server.sh | 2 +- tests/test_find_duplicate_files.sh | 2 +- tests/test_getent.sh | 2 +- tests/test_git_check_branches_upstream.sh | 2 +- tests/test_json_to_xml.sh | 2 +- tests/test_quay_show_tags.sh | 2 +- tests/test_serf_event_handler.sh | 2 +- tests/test_xml_to_json.sh | 2 +- tests/utils.sh | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/all.sh b/tests/all.sh index 8206b4347..15e3af28d 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -20,7 +20,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . "$srcdir/utils.sh" # imported by utils.sh above -#. "$srcdir/../bash-tools/utils.sh" +#. "$srcdir/../bash-tools/lib/utils.sh" export PROJECT=pytools diff --git a/tests/test_center.sh b/tests/test_center.sh index 8663b5cee..976466e1a 100755 --- a/tests/test_center.sh +++ b/tests/test_center.sh @@ -22,7 +22,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "Testing center.py" diff --git a/tests/test_docker.sh b/tests/test_docker.sh index 77ea4530e..bcdd82e40 100755 --- a/tests/test_docker.sh +++ b/tests/test_docker.sh @@ -20,7 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." . "bash-tools/docker.sh" -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "Docker Image" diff --git a/tests/test_docker_registry_show_tags.sh b/tests/test_docker_registry_show_tags.sh index ba4fe96c9..d3d72b932 100755 --- a/tests/test_docker_registry_show_tags.sh +++ b/tests/test_docker_registry_show_tags.sh @@ -21,7 +21,7 @@ cd "$srcdir2/.." . "tests/utils.sh" -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" srcdir="$srcdir2" diff --git a/tests/test_dockerfiles_check_git_branches.sh b/tests/test_dockerfiles_check_git_branches.sh index 040b0a38d..90bf1ac7a 100755 --- a/tests/test_dockerfiles_check_git_branches.sh +++ b/tests/test_dockerfiles_check_git_branches.sh @@ -21,7 +21,7 @@ cd "$srcdir/.." . "tests/utils.sh" -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "Testing Dockerfiles Check Git branches" diff --git a/tests/test_dockerfiles_check_git_tags.sh b/tests/test_dockerfiles_check_git_tags.sh index 51dd21ac9..372076138 100755 --- a/tests/test_dockerfiles_check_git_tags.sh +++ b/tests/test_dockerfiles_check_git_tags.sh @@ -21,7 +21,7 @@ cd "$srcdir/.." . "tests/utils.sh" -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "Testing Dockerfiles Check Git Tags" diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index 54d83458a..d625bf2db 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -21,7 +21,7 @@ cd "$srcdir/.." . "tests/utils.sh" -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "Testing DockerHub Show Tags" diff --git a/tests/test_dockerhub_show_tags.sh b/tests/test_dockerhub_show_tags.sh index df24d1fe0..71f424a3f 100755 --- a/tests/test_dockerhub_show_tags.sh +++ b/tests/test_dockerhub_show_tags.sh @@ -21,7 +21,7 @@ cd "$srcdir/.." . "tests/utils.sh" -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "Testing DockerHub Show Tags" diff --git a/tests/test_find_active_server.sh b/tests/test_find_active_server.sh index 1eb42e1d9..74bb8e494 100755 --- a/tests/test_find_active_server.sh +++ b/tests/test_find_active_server.sh @@ -20,7 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." -. bash-tools/utils.sh +. bash-tools/lib/utils.sh set +e +o pipefail diff --git a/tests/test_find_duplicate_files.sh b/tests/test_find_duplicate_files.sh index 33de6d0a6..807508f52 100755 --- a/tests/test_find_duplicate_files.sh +++ b/tests/test_find_duplicate_files.sh @@ -20,7 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "find_duplicate_files.py" diff --git a/tests/test_getent.sh b/tests/test_getent.sh index 600f4df43..d2b87ddfe 100755 --- a/tests/test_getent.sh +++ b/tests/test_getent.sh @@ -19,7 +19,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." -. ./bash-tools/utils.sh +. ./bash-tools/lib/utils.sh section "Getent" diff --git a/tests/test_git_check_branches_upstream.sh b/tests/test_git_check_branches_upstream.sh index e3b5eea5d..8ae7a55ae 100755 --- a/tests/test_git_check_branches_upstream.sh +++ b/tests/test_git_check_branches_upstream.sh @@ -21,7 +21,7 @@ cd "$srcdir/.." . "tests/utils.sh" -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "Testing Git check branches upstream" diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 1500aa928..81a8f2669 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -20,7 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir"; . utils.sh -. ../bash-tools/utils.sh +. ../bash-tools/lib/utils.sh section "JSON => XML" diff --git a/tests/test_quay_show_tags.sh b/tests/test_quay_show_tags.sh index 711ddb42e..c9c55ad62 100755 --- a/tests/test_quay_show_tags.sh +++ b/tests/test_quay_show_tags.sh @@ -21,7 +21,7 @@ cd "$srcdir/.." . "tests/utils.sh" -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "Testing Quay.io Show Tags" diff --git a/tests/test_serf_event_handler.sh b/tests/test_serf_event_handler.sh index ad8218d32..00acd0fe7 100755 --- a/tests/test_serf_event_handler.sh +++ b/tests/test_serf_event_handler.sh @@ -21,7 +21,7 @@ cd "$srcdir/.." . "tests/utils.sh" -. "bash-tools/utils.sh" +. "bash-tools/lib/utils.sh" section "Testing Serf Event Handler" diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index dcedb8937..17d15b2c7 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -20,7 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir"; . utils.sh -. ../bash-tools/utils.sh +. ../bash-tools/lib/utils.sh section "XML => JSON" diff --git a/tests/utils.sh b/tests/utils.sh index 3734502c0..e31af5f4b 100755 --- a/tests/utils.sh +++ b/tests/utils.sh @@ -18,7 +18,7 @@ set -eu srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . "$srcdir2/excluded.sh" -. "$srcdir2/../bash-tools/utils.sh" +. "$srcdir2/../bash-tools/lib/utils.sh" . "$srcdir2/../bash-tools/docker.sh" srcdir="$srcdir2" From 57d2e0194e039b7219ed5a44125a869c1c532e83 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 31 Jul 2019 19:09:02 +0100 Subject: [PATCH 0277/2779] updated bash tools docker references --- tests/test_docker.sh | 2 +- tests/test_hbase.sh | 2 +- tests/utils.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_docker.sh b/tests/test_docker.sh index bcdd82e40..ddb02baa7 100755 --- a/tests/test_docker.sh +++ b/tests/test_docker.sh @@ -19,7 +19,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." -. "bash-tools/docker.sh" +. "bash-tools/lib/docker.sh" . "bash-tools/lib/utils.sh" section "Docker Image" diff --git a/tests/test_hbase.sh b/tests/test_hbase.sh index fdcc7d7b6..ebde33a96 100755 --- a/tests/test_hbase.sh +++ b/tests/test_hbase.sh @@ -21,7 +21,7 @@ srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir2/.." . "$srcdir2/utils.sh" -. "$srcdir2/../bash-tools/docker.sh" +. "$srcdir2/../bash-tools/lib/docker.sh" srcdir="$srcdir2" diff --git a/tests/utils.sh b/tests/utils.sh index e31af5f4b..01cd59ac9 100755 --- a/tests/utils.sh +++ b/tests/utils.sh @@ -19,7 +19,7 @@ srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" . "$srcdir2/excluded.sh" . "$srcdir2/../bash-tools/lib/utils.sh" -. "$srcdir2/../bash-tools/docker.sh" +. "$srcdir2/../bash-tools/lib/docker.sh" srcdir="$srcdir2" From 13e5a9516208afd55ed188039cae930e977ed81b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Aug 2019 16:15:41 +0100 Subject: [PATCH 0278/2779] updated travis_last_log.py --- travis_last_log.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/travis_last_log.py b/travis_last_log.py index 54f3ccc5f..063c06486 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -50,7 +50,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from __future__ import unicode_literals +# this breaks regex on Python 2.7 with UnicodeDecodeError traceback from the Travis CI output +#from __future__ import unicode_literals import json import logging From 29899e1b184b4c541c5c0bc133a4f9a057076404 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 3 Aug 2019 13:48:22 +0100 Subject: [PATCH 0279/2779] updated travis_debug_session.py --- travis_debug_session.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/travis_debug_session.py b/travis_debug_session.py index c442a26cd..938ea5b2f 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -26,7 +26,7 @@ If specifying a --repo be aware the API is case sensitive for repo names As a convenience you may supply either job id or repo as an argument without any switch and it'll infer it as a repo if -if contains a slash but no url (eg. HariSekhon/nagios-plugins) otherwise it'll assume it's a job id, strip any leading +if contains a slash but no url (eg. HariSekhon/Nagios-Plugins) otherwise it'll assume it's a job id, strip any leading URL so you can simply paste the path to a failing build and it'll just work. The switch versions of --job-id and --repo take priority as they're more explicit @@ -124,7 +124,7 @@ def process_options(self): # self.usage('--travis-token option or ' + # '$TRAVIS_TOKEN environment variable required to authenticate to the API') if self.args: - # assume arg is a repo in form of HariSekhon/nagios-plugins but do not use url which we are more likely to + # assume arg is a repo in form of HariSekhon/Nagios-Plugins but do not use url which we are more likely to # have pasted a travis-ci url to a job, see a few lines further down if '/' in self.args[0]: if not self.repo: @@ -135,7 +135,7 @@ def process_options(self): self.job_id = self.args[0] if self.job_id: # convenience to be able to lazily paste a URL like the following and still have it extract the job_id - # https://travis-ci.org/HariSekhon/nagios-plugins/jobs/283840596#L1079 + # https://travis-ci.org/HariSekhon/Nagios-Plugins/jobs/283840596#L1079 self.job_id = self.job_id.split('/')[-1].split('#')[0] validate_chars(self.job_id, 'job id', '0-9') elif self.repo: @@ -210,7 +210,7 @@ def parse_latest_failed_build(self, content): + " Either the specified repo '{0}' doesn't exist".format(self.repo) + " or no builds have happened yet?" + " Also remember the repo is case sensitive, for example 'harisekhon/nagios-plugins' returns this" - + " blank build set whereas 'HariSekhon/nagios-plugins' succeeds" + + " blank build set whereas 'HariSekhon/Nagios-Plugins' succeeds" + " in returning latest builds information" ) builds = json_data['builds'] From 2ca457b182fc2a23c2fe31870aa11ff801f712bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 3 Aug 2019 13:48:26 +0100 Subject: [PATCH 0280/2779] updated travis_last_log.py --- travis_last_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis_last_log.py b/travis_last_log.py index 063c06486..b9b6843f2 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -218,7 +218,7 @@ def parse_builds(self, content): + " Either the specified repo '{0}' doesn't exist".format(self.repo) + " or no builds have happened yet?" + " Also remember the repo is case sensitive, for example 'harisekhon/nagios-plugins' returns this" - + " blank build set whereas 'HariSekhon/nagios-plugins' succeeds" + + " blank build set whereas 'HariSekhon/Nagios-Plugins' succeeds" + " in returning latest builds information" ) builds = json_data['builds'] From 53187881c48c0b3c2ce5e350929465c386137777 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 3 Aug 2019 13:50:46 +0100 Subject: [PATCH 0281/2779] added auto-repo completion from TRAVIS_USER --- travis_debug_session.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/travis_debug_session.py b/travis_debug_session.py index 938ea5b2f..51847ed46 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -64,7 +64,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.3' +__version__ = '0.8.4' class TravisDebugSession(CLI): @@ -139,7 +139,12 @@ def process_options(self): self.job_id = self.job_id.split('/')[-1].split('#')[0] validate_chars(self.job_id, 'job id', '0-9') elif self.repo: - self.repo = re.sub(r'https://travis-ci\.org/', '', self.repo) + self.repo = re.sub(r'https?://travis-ci\.org/', '', self.repo) + travis_user = os.getenv('TRAVIS_USER') + if '/' not in self.repo: + self.repo = '/' + self.repo + if self.repo[0] == '/' and travis_user: + self.repo = travis_user + self.repo validate_chars(self.repo, 'repo', r'\/\w\.-') else: self.usage('--job-id / --repo not specified') From 017308e60b5d5a891fe22b579795370ef382adfb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 3 Aug 2019 14:09:59 +0100 Subject: [PATCH 0282/2779] updated Makefile --- Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Makefile b/Makefile index b73c71272..b610f49fd 100755 --- a/Makefile +++ b/Makefile @@ -29,6 +29,8 @@ # =================== +REPO := HariSekhon/DevOps-Python-tools + ifneq ("$(wildcard bash-tools/Makefile.in)", "") include bash-tools/Makefile.in endif @@ -171,6 +173,14 @@ test2: cd pylib && $(MAKE) test2 tests/all.sh +.PHONY: travis +travis: + travis_last_log.py $(REPO) + +.PHONY: travis-debug +travis-debug: + travis_debug_session.py $(REPO) + .PHONY: install install: @echo "No installation needed, just add '$(PWD)' to your \$$PATH" From 424751a8cf2d66a26fb8bbdb25d4edb21e74a527 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Aug 2019 17:13:07 +0100 Subject: [PATCH 0283/2779] updated hbase_flush_tables.py --- hbase_flush_tables.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hbase_flush_tables.py b/hbase_flush_tables.py index 4da3bb492..c4f43ef8d 100755 --- a/hbase_flush_tables.py +++ b/hbase_flush_tables.py @@ -27,6 +27,8 @@ The Thrift API doesn't support this action so it uses the HBase shell locally which must be in the $PATH +There is also a shell script version of this in the adjacent DevOps-Perl-Tools repo + Tested on Hortonworks HDP 2.3 (HBase 1.1.2) and Apache HBase 1.0.3, 1.1.6, 1.2.1, 1.2.2, 1.3.1 """ From 75684412c4f31dacb1b6c871202d5ffcb6e72b18 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Aug 2019 14:19:48 +0100 Subject: [PATCH 0284/2779] updated Makefile --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index b610f49fd..9060eb78e 100755 --- a/Makefile +++ b/Makefile @@ -31,6 +31,8 @@ REPO := HariSekhon/DevOps-Python-tools +CODE_FILES := $(shell find . -type f -name '*.py' -o -type f -name '*.sh' | grep -v -e bash-tools -e /pylib/) + ifneq ("$(wildcard bash-tools/Makefile.in)", "") include bash-tools/Makefile.in endif From 6c05db2794cc94c0903f81af7763b4052539911e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Sep 2019 17:25:13 +0100 Subject: [PATCH 0285/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9a54f2060..ca87ee8c6 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9a54f20609c45c0624120866ef2eb309a60acb9e +Subproject commit ca87ee8c6d31a87bb5747ec4a92ebb52b8ff4336 From 3ba7131f8287d849371c80f6ccbc8c0210db60f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Sep 2019 17:25:13 +0100 Subject: [PATCH 0286/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5248361e5..d8ccc0824 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5248361e58ce15b0c2465f2c52a3ba6e31f951f9 +Subproject commit d8ccc082416c2934d0dc394bbc9f6a52b822b484 From 0f14977c3d4a6117a1ce697a95ceb1a5c44a7c27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Sep 2019 17:28:12 +0100 Subject: [PATCH 0287/2779] updated .travis.yml --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9d5678f9a..08da1e02c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,10 @@ python: # - "pypy" # - "pypy3" +os: + - linux + - osx + dist: trusty sudo: required From 4df1c3a0db410f3c0555a6e962ad448e7de4c229 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Sep 2019 17:46:33 +0100 Subject: [PATCH 0288/2779] updated .travis.yml --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 08da1e02c..b5f93be91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,9 +26,10 @@ python: # - "pypy" # - "pypy3" -os: - - linux - - osx +# Mac Python 2.7 not available - see https://travis-ci.org/HariSekhon/pylib/jobs/584226228 +#os: +# - linux +# - osx dist: trusty From c1b2ec1a600fa4fde15941e27d24d34664f913aa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Sep 2019 17:46:37 +0100 Subject: [PATCH 0289/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ca87ee8c6..f625da51c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ca87ee8c6d31a87bb5747ec4a92ebb52b8ff4336 +Subproject commit f625da51c828878a19c4a0bb2ff6cf16d91e7222 From ed24aea60bbf05059d7b4eea1a63dd1d254de215 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Sep 2019 17:46:37 +0100 Subject: [PATCH 0290/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d8ccc0824..3de2416fe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d8ccc082416c2934d0dc394bbc9f6a52b822b484 +Subproject commit 3de2416fe522c1407b2c62189f92b9e4631ca4fc From f0a983335b481e721acde58a13434fe8b4e5814f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Sep 2019 10:06:14 +0100 Subject: [PATCH 0291/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f625da51c..d80207fd7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f625da51c828878a19c4a0bb2ff6cf16d91e7222 +Subproject commit d80207fd75914db05a57b848b327418f27dd06cb From 22f85a560efd740887b190a029f0042f16aaae6d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Sep 2019 10:06:14 +0100 Subject: [PATCH 0292/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3de2416fe..0af0f48c2 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3de2416fe522c1407b2c62189f92b9e4631ca4fc +Subproject commit 0af0f48c294274655105aacf5cf932a33ab3b2d8 From da4f5e903e07a8798c0da36d718715a3b013d29d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Sep 2019 10:06:30 +0100 Subject: [PATCH 0293/2779] updated Makefile --- Makefile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Makefile b/Makefile index b610f49fd..206ea7565 100755 --- a/Makefile +++ b/Makefile @@ -173,14 +173,6 @@ test2: cd pylib && $(MAKE) test2 tests/all.sh -.PHONY: travis -travis: - travis_last_log.py $(REPO) - -.PHONY: travis-debug -travis-debug: - travis_debug_session.py $(REPO) - .PHONY: install install: @echo "No installation needed, just add '$(PWD)' to your \$$PATH" From 2d77491e7915a80979e1165e3021dace339f6272 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 15 Sep 2019 10:28:45 +0100 Subject: [PATCH 0294/2779] updated Makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 68bf84bf5..7bd2ccdbf 100755 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ build: .PHONY: init init: git submodule update --init --recursive - + .PHONY: python python: cd pylib && $(MAKE) @@ -68,7 +68,7 @@ python: else \ $(MAKE) parquet-tools; \ fi - + # only install pip packages not installed via system packages #$(SUDO_PIP) pip install --upgrade -r requirements.txt #$(SUDO_PIP) pip install -r requirements.txt @@ -98,7 +98,7 @@ python: #$(SUDO_PIP) pip install impyla # must downgrade happybase library to work on Python 2.6 #if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi - + # Python >= 2.7 - won't build on 2.6, handle separately and accept failure $(SUDO_PIP) pip install "ipython[notebook]" || : @echo From 56f0acb9f2aebc0cbcf184d5847762e658336bad Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 15 Sep 2019 11:36:14 +0100 Subject: [PATCH 0295/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d80207fd7..1902a03a9 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d80207fd75914db05a57b848b327418f27dd06cb +Subproject commit 1902a03a9bef5078fcaee8b517c14147a599221e From 9ba26ce94c814755ab37bfbdd5c509ada2e4399c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 15 Sep 2019 11:36:14 +0100 Subject: [PATCH 0296/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0af0f48c2..639ba3dc2 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0af0f48c294274655105aacf5cf932a33ab3b2d8 +Subproject commit 639ba3dc21ed78ead68e2877b5c4013876d1109f From adad652b629f6fd56c98988739e47558c9aebed5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 15 Sep 2019 11:37:21 +0100 Subject: [PATCH 0297/2779] updated .travis.yml --- .travis.yml | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5f93be91..ccbeb9cea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,24 +13,30 @@ # https://www.linkedin.com/in/harisekhon # -language: python -python: -# - "2.6" - - "2.7" -# MySQL in lib doesn't build from pip in Python 3 -# - "3.2" -# - "3.3" -# - "3.4" -# - "3.5" -# python-krbV fails to compile on PyPy -# - "pypy" -# - "pypy3" - # Mac Python 2.7 not available - see https://travis-ci.org/HariSekhon/pylib/jobs/584226228 #os: # - linux # - osx +matrix: + include: + - os: linux + language: python + python: + # - "2.6" + - "2.7" + # MySQL in lib doesn't build from pip in Python 3 + # - "3.2" + # - "3.3" + # - "3.4" + # - "3.5" + # python-krbV fails to compile on PyPy + # - "pypy" + # - "pypy3" + # workaround is to use generic and install to system python + - os: osx + language: generic + dist: trusty sudo: required From 0ead192b0a01cbcf424b8fc853a84495c0d62b24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2019 12:23:39 +0100 Subject: [PATCH 0298/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1902a03a9..fb38074d2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1902a03a9bef5078fcaee8b517c14147a599221e +Subproject commit fb38074d2ff3c68cac1cd2ee3f2f45345716ff96 From 40fef3f65b8e064349ca2320290658ff7e1d16c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2019 12:23:39 +0100 Subject: [PATCH 0299/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 639ba3dc2..e39dbbee3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 639ba3dc21ed78ead68e2877b5c4013876d1109f +Subproject commit e39dbbee344aaa69526978c82a1be3ce86b412fe From 6a5781936abe3cb65d1bdc15b94d37e45690df80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2019 13:17:30 +0100 Subject: [PATCH 0300/2779] added .github/ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..b951f3244 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,7 @@ +Please be as specific as possible when raising an issue. + +- what were you expecting +- what was the result +- what was the output - if running a CLI program please include the full debug output when running with program by using the `--debug` or `-v -v -v` switches for most tools, or setting `export DEBUG=1` environment variable, which also works with the shell scripts. + +You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively. From a9a814dc70f70f558da786167d6453ca85ea29a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2019 18:57:30 +0100 Subject: [PATCH 0301/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fb38074d2..45258641c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fb38074d2ff3c68cac1cd2ee3f2f45345716ff96 +Subproject commit 45258641c9624ef3e8439e79e0bdba44f473b177 From 4fad4e7c339959bf1214d2fe27eca0eea90086d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2019 18:57:30 +0100 Subject: [PATCH 0302/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e39dbbee3..63f420bc6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e39dbbee344aaa69526978c82a1be3ce86b412fe +Subproject commit 63f420bc6af67e7c1e46d9772982aba589d1019a From c28ae4906462da8f705028603fb50480c5c3e661 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 11:45:37 +0100 Subject: [PATCH 0303/2779] updated setup/brew-packages.txt --- setup/brew-packages.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index f11d663a9..75a9dd255 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -17,5 +17,6 @@ # for anonymize_parallel.sh parallel +parquet-tools snappy yamllint From 232ec71c2ad1df1c814bfee9814c5bdf20a3349d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 11:47:07 +0100 Subject: [PATCH 0304/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7bd2ccdbf..55dc9d8ab 100755 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ ifneq ("$(wildcard bash-tools/Makefile.in)", "") include bash-tools/Makefile.in endif -PARQUET_VERSION=1.5.0 +PARQUET_VERSION=1.6.0 .PHONY: build build: From 2dd0dcc2ecae717d317f38c6fc9053ff3c833253 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 11:59:50 +0100 Subject: [PATCH 0305/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 55dc9d8ab..57e350632 100755 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ python: .PHONY: parquet-tools parquet-tools: - if ! [ -d "parquet-tools-$(PARQUET_VERSION)" ]; then \ + if ! command -v parquet-cat && ! [ -d "parquet-tools-$(PARQUET_VERSION)" ]; then \ wget -t 100 --retry-connrefused -c -O "parquet-tools-$(PARQUET_VERSION)-bin.zip" "http://search.maven.org/remotecontent?filepath=com/twitter/parquet-tools/$(PARQUET_VERSION)/parquet-tools-$(PARQUET_VERSION)-bin.zip"; \ unzip "parquet-tools-$(PARQUET_VERSION)-bin.zip"; \ fi From 34d94ee906fa45c193c6b7d64fa7ac866edb7c08 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 12:31:44 +0100 Subject: [PATCH 0306/2779] updated Makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 57e350632..e04d7720d 100755 --- a/Makefile +++ b/Makefile @@ -37,13 +37,13 @@ ifneq ("$(wildcard bash-tools/Makefile.in)", "") include bash-tools/Makefile.in endif -PARQUET_VERSION=1.6.0 +PARQUET_VERSION=1.5.0 .PHONY: build build: - @echo ============= + @echo ========================= @echo DevOps Python Tools Build - @echo ============= + @echo ========================= $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi From acdc08a895de4f375961921c14bd1bf0b8546851 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 12:47:10 +0100 Subject: [PATCH 0307/2779] set reverse globbing on local parquet tools and only for dirs --- validate_parquet.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/validate_parquet.py b/validate_parquet.py index 1db908320..71894645f 100755 --- a/validate_parquet.py +++ b/validate_parquet.py @@ -65,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.0' +__version__ = '0.9.1' class ParquetValidatorTool(CLI): @@ -80,9 +80,10 @@ def __init__(self): self.valid_parquet_msg = ' => Parquet OK' self.invalid_parquet_msg = ' => Parquet INVALID' self.exclude = None - for _ in glob.glob(os.path.join(os.path.dirname(__file__), 'parquet-tools-*')): - log.debug('adding %s to $PATH' % _) - os.environ['PATH'] += ':' + os.path.abspath(_) + for _ in reversed(glob.glob(os.path.join(os.path.dirname(__file__), 'parquet-tools-*'))): + if os.path.isdir(_): + log.debug('adding %s to $PATH' % _) + os.environ['PATH'] += ':' + os.path.abspath(_) def add_options(self): self.add_opt('-e', '--exclude', metavar='regex', default=os.getenv('EXCLUDE'), From a338f4ec351e5ffe20b114f261a0b489e764c4df Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 12:52:13 +0100 Subject: [PATCH 0308/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fb38074d2..61e2bb5cc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fb38074d2ff3c68cac1cd2ee3f2f45345716ff96 +Subproject commit 61e2bb5cc27c297e54cde54ed81073a01cc8176b From 485b1aa381cad7c5a6233dc9e983d2a19a199244 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 12:55:01 +0100 Subject: [PATCH 0309/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 61e2bb5cc..9b0c408fe 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 61e2bb5cc27c297e54cde54ed81073a01cc8176b +Subproject commit 9b0c408fefd7ce07d024de663c77b2a58fbe2277 From 1644a562cf23863dfbb63b154335f57b8d96afd8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 14:25:07 +0100 Subject: [PATCH 0310/2779] updated setup/deb-packages-dev.txt --- setup/deb-packages-dev.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index e6936942c..9e9899331 100644 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -16,3 +16,6 @@ libldap2-dev # needed to build python-snappy for avro module libsnappy-dev + +# needed to build python-krbV +libsasl2-dev From 63c17288d97a8b2a42bf0b921087c1a024922f27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 14:25:29 +0100 Subject: [PATCH 0311/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9b0c408fe..0a7acc6ad 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9b0c408fefd7ce07d024de663c77b2a58fbe2277 +Subproject commit 0a7acc6ad4e5b4d0173d121395eeb07cfff6ea54 From 11aeca753c71283cb53a3e5ddd8e727a4ccd4adf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 14:33:43 +0100 Subject: [PATCH 0312/2779] updated requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7aa194833..3b3f3a0ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,8 @@ python-ldap>=3.2.0 # PyLint fails some checks in Travis that pass locally #pyhs2>=0.6.0 #python-jenkins>=0.4.13 -python-krbV>=1.0.90 +# pulled in automatically by snakebite[kerberos] +#python-krbV>=1.0.90 # needed by avro python-snappy>=0.5 sasl>=0.2.1 From e4654770620499dacc8137253443e8751243dc7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 17:41:48 +0100 Subject: [PATCH 0313/2779] updated setup/brew-packages.txt --- setup/brew-packages.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index 75a9dd255..fa25317ba 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -20,3 +20,6 @@ parallel parquet-tools snappy yamllint + +# Mac already provides unzip - brew is keg only and doesn't link anyway to avoid clashes +#unzip From 835f386710fdee73ef78fcc3922c28f99fa6661b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 18:01:59 +0100 Subject: [PATCH 0314/2779] updated Makefile --- Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Makefile b/Makefile index e04d7720d..d1878b249 100755 --- a/Makefile +++ b/Makefile @@ -37,8 +37,6 @@ ifneq ("$(wildcard bash-tools/Makefile.in)", "") include bash-tools/Makefile.in endif -PARQUET_VERSION=1.5.0 - .PHONY: build build: @echo ========================= @@ -111,10 +109,7 @@ python: .PHONY: parquet-tools parquet-tools: - if ! command -v parquet-cat && ! [ -d "parquet-tools-$(PARQUET_VERSION)" ]; then \ - wget -t 100 --retry-connrefused -c -O "parquet-tools-$(PARQUET_VERSION)-bin.zip" "http://search.maven.org/remotecontent?filepath=com/twitter/parquet-tools/$(PARQUET_VERSION)/parquet-tools-$(PARQUET_VERSION)-bin.zip"; \ - unzip "parquet-tools-$(PARQUET_VERSION)-bin.zip"; \ - fi + @BIN='.' bash-tools/setup/install_parquet-tools.sh .PHONY: apk-packages-extra apk-packages-extra: From 18365dd97b0f10cc1a97beca7efec814d81fe7e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 18:02:11 +0100 Subject: [PATCH 0315/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0a7acc6ad..1098cf931 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0a7acc6ad4e5b4d0173d121395eeb07cfff6ea54 +Subproject commit 1098cf931788006da9b53537d3a5b5064b18dde9 From 92340d3fa1a58ac69d08a6382946ecb9575accec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Sep 2019 18:02:11 +0100 Subject: [PATCH 0316/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 63f420bc6..36ab832ce 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 63f420bc6af67e7c1e46d9772982aba589d1019a +Subproject commit 36ab832ce56106ae6494268861419aa7fb37b30c From 03205f3d8d9963b04d643d05c5d8329aecf4c020 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Sep 2019 14:19:12 +0100 Subject: [PATCH 0317/2779] updated .gitignore --- .gitignore | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/.gitignore b/.gitignore index 525ea88b7..8909eea29 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # This doesn't actually work on already committed files, git update-index --assume-unchanged is set in Makefile instead anonymize_custom.conf + solr-env.sh spark-deps.zip *.pyc @@ -22,3 +23,135 @@ parquet-tools-*-bin.zip tests/docker/registry.crt tests/docker/registry.csr tests/docker/registry.key + +# ============================================================================ # +# Created by https://www.gitignore.io/api/python +# Edit at https://www.gitignore.io/?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# End of https://www.gitignore.io/api/python From 2bb74761c7a53f365409bfdae1b8f7c31df7ba0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Sep 2019 14:20:24 +0100 Subject: [PATCH 0318/2779] updated .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 8909eea29..a5b101bfe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +# vim: filetype=conf + # This doesn't actually work on already committed files, git update-index --assume-unchanged is set in Makefile instead anonymize_custom.conf From d0e73d648db7896f69888643ce41a1832038b0bd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Sep 2019 11:21:24 +0100 Subject: [PATCH 0319/2779] updated requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 3b3f3a0ea..9ae833fa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ avro>=1.8.1 +awscli>=1.16.241 #bitarray>=0.8.1 #cassandra-driver>=3.6.0 dicttoxml>=1.7.4 From 4211778b5d35ddf30a23fcfbbc75e5253084c4cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Sep 2019 14:09:01 +0100 Subject: [PATCH 0320/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e08ff3d3..07dec23e6 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ Then add the Jython install bin directory to the $PATH or specify the full path #### Configuration for Strict Domain / FQDN validation #### -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/harisekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like ```.local``` or ```.intranet``` (both supported by default). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/harisekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). #### Python SSL certificate verification problems From c54eebdd3279e9ec707a5cebf80513a9aa07115a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Sep 2019 22:29:23 +0100 Subject: [PATCH 0321/2779] updated .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a5b101bfe..f0ca6569a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ spark-deps.zip *.pyc *.pyo *.jyc +*.jyo *.avro *.parquet *.log From 8e5ce7d3bea028f146abfcf87c1ac4772a374dbf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Sep 2019 10:30:35 +0100 Subject: [PATCH 0322/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d1878b249..7fdac844c 100755 --- a/Makefile +++ b/Makefile @@ -171,7 +171,7 @@ test2: tests/all.sh .PHONY: install -install: +install: build @echo "No installation needed, just add '$(PWD)' to your \$$PATH" .PHONY: clean From 1480d2e5907b828e29f70721a019f1d4925f1503 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Sep 2019 11:18:34 +0100 Subject: [PATCH 0323/2779] moved snakebite[kerberos] to best effort since python-krbV is unmaintained and no longer builds on Mac --- Makefile | 5 +++++ requirements.txt | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7fdac844c..7d5618100 100755 --- a/Makefile +++ b/Makefile @@ -72,6 +72,11 @@ python: #$(SUDO_PIP) pip install -r requirements.txt @bash-tools/python_pip_install_if_absent.sh requirements.txt + # python-krbV dependency doesn't build on Mac any more and is unmaintained + # python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit + #@bash-tools/python_pip_install.sh snakebite[kerberos] || : + @bash-tools/python_pip_install.sh snakebite[kerberos] || : + # for impyla #$(SUDO_PIP) pip install --upgrade setuptools || : # diff --git a/requirements.txt b/requirements.txt index 9ae833fa3..257d774f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,10 @@ python-ldap>=3.2.0 python-snappy>=0.5 sasl>=0.2.1 sh>=1.12.14 -snakebite[kerberos]>=2.11.0 +# pulls in python-KrbV as a dependency which doesn't build on Mac any more +# moved to Makefile as best effort +#snakebite[kerberos]>=2.11.0 +snakebite>=2.11.0 thrift-sasl>=0.2.1 thrift>=0.9.3 thriftpy>=0.3.9 From 5d48cde51ebc6e425b6a83efbef56ff0f1a237ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Sep 2019 12:03:26 +0100 Subject: [PATCH 0324/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1098cf931..783a11795 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1098cf931788006da9b53537d3a5b5064b18dde9 +Subproject commit 783a117953fbd93bd334dd4c0e0dcea9040b3c3f From 40f397174743568f3fe9ea11123c7f1e7f9dd2f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Sep 2019 12:03:26 +0100 Subject: [PATCH 0325/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 36ab832ce..a1472f403 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 36ab832ce56106ae6494268861419aa7fb37b30c +Subproject commit a1472f403cc9d16db13147aad62796403eb117d2 From 0a62a6efaef7eae4503607196a51db4aa5f04e42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Sep 2019 12:39:28 +0100 Subject: [PATCH 0326/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 783a11795..0a14375e4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 783a117953fbd93bd334dd4c0e0dcea9040b3c3f +Subproject commit 0a14375e425bfd6b87039ab593849f6998bd65d5 From 73e96e9176bcb7ade288e42a4bf4f80d7e7f79af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 00:52:13 +0100 Subject: [PATCH 0327/2779] tweaks to pass python 3 validation --- hadoop_hdfs_files_native_checksums.jy | 6 ++-- hadoop_hdfs_files_stats.jy | 12 +++---- hadoop_hdfs_time_block_reads.jy | 46 +++++++++++++-------------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/hadoop_hdfs_files_native_checksums.jy b/hadoop_hdfs_files_native_checksums.jy index 29ed7e80e..a1bc67622 100755 --- a/hadoop_hdfs_files_native_checksums.jy +++ b/hadoop_hdfs_files_native_checksums.jy @@ -15,7 +15,7 @@ from __future__ import print_function from __future__ import unicode_literals __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' # pylint: disable=wrong-import-position import os @@ -142,7 +142,7 @@ class HDFSChecksumReader(object): printerr("Failed to get HDFS path object for file " + filename, 1) continue if not self.filesystem.exists(path): - #raise IOError, "HDFS File not found: %s" % filename + #raise IOError("HDFS File not found: %s" % filename) printerr("HDFS file/dir not found: %s" % filename, 1) continue self.print_checksum_recurse(filename, path) @@ -161,7 +161,7 @@ class HDFSChecksumReader(object): except: printerr(get_jython_exception().split("\n")[0], 1) else: - raise IOError, ">>> %s is not a file or directory" % filename + raise IOError(">>> %s is not a file or directory" % filename) def print_checksum(self, filename, path): """ Prints the HDFS checksum for the given file + fs path object """ diff --git a/hadoop_hdfs_files_stats.jy b/hadoop_hdfs_files_stats.jy index ce7ae265c..69132daee 100755 --- a/hadoop_hdfs_files_stats.jy +++ b/hadoop_hdfs_files_stats.jy @@ -15,7 +15,7 @@ from __future__ import print_function from __future__ import unicode_literals __author__ = 'Hari Sekhon' -__version__ = '0.9.1' +__version__ = '0.9.2' # pylint: disable=wrong-import-position import os @@ -146,7 +146,7 @@ class HDFSBlockCounter(object): try: self.client = self.filesystem.getClient() except: - raise Exception, "Failed to create hdfs client: %s" % get_jython_exception() + raise Exception("Failed to create hdfs client: %s" % get_jython_exception()) #in = client.DFSDataInputStream(self.filehandle) @@ -178,7 +178,7 @@ class HDFSBlockCounter(object): printerr("Failed to get HDFS path object for file " + filename, 1) continue if not self.filesystem.exists(path): - #raise IOError, "HDFS File not found: %s" % filename + #raise IOError("HDFS File not found: %s" % filename) printerr("HDFS file/dir not found: %s" % filename, 1) continue self.recurse_path(filename, path) @@ -213,7 +213,7 @@ class HDFSBlockCounter(object): except: printerr(get_jython_exception().split("\n")[0], 1) else: - raise IOError, ">>> %s is not a file or directory" % filename + raise IOError(">>> %s is not a file or directory" % filename) def print_file_block_counts(self, filename, path): @@ -226,9 +226,9 @@ class HDFSBlockCounter(object): try: self.filehandle = self.client.open(filename) except: - raise IOError, "Failed to get client filehandle to HDFS file %s: %s" % (filename, get_jython_exception()) + raise IOError("Failed to get client filehandle to HDFS file %s: %s" % (filename, get_jython_exception())) if self.filehandle is None: - raise IOError, "Failed to get client filehandle to HDFS file %s" % filename + raise IOError("Failed to get client filehandle to HDFS file %s" % filename) try: filestatus = self.filesystem.getFileStatus(path) diff --git a/hadoop_hdfs_time_block_reads.jy b/hadoop_hdfs_time_block_reads.jy index a212c3e6b..e340cff7c 100755 --- a/hadoop_hdfs_time_block_reads.jy +++ b/hadoop_hdfs_time_block_reads.jy @@ -23,7 +23,7 @@ from __future__ import print_function from __future__ import unicode_literals __author__ = 'Hari Sekhon' -__version__ = '0.9.0' +__version__ = '0.9.1' import os @@ -95,7 +95,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), 'pylib')) try: from harisekhon.utils import jython_only, die, printerr, ERRORS, \ isJavaOOM, java_oom_fix_msg, log_jython_exception, get_jython_exception -except ImportError, e: +except ImportError as e: print('module import failed: %s' % e, file=sys.stderr) sys.exit(4) @@ -105,14 +105,14 @@ jython_only() import jarray try: from java.nio import ByteBuffer -except ImportError, e: +except ImportError as e: die("Couldn't find java.nio class, not running inside Jython?") try: from org.apache.hadoop.conf import Configuration #from org.apache.hadoop.fs import FileSystem from org.apache.hadoop.fs import Path from org.apache.hadoop.hdfs import DistributedFileSystem -except ImportError, e: +except ImportError as e: die("Couldn't find Hadoop Java classes, try: jython -J-cp `hadoop classpath` hadoop_hdfs_time_block_reads.jy ") def usage(*msg): @@ -127,7 +127,7 @@ def main(): try: opts, args = getopt.gnu_getopt(sys.argv[1:], "hamo", ["help", "usage", "all-blocks", "multiple-blocks", "one-block-per-DN"]) - except getopt.GetoptError, e: + except getopt.GetoptError as e: usage("error: %s" % e) all_blocks = False multiple_blocks = False @@ -153,10 +153,10 @@ def main(): filelist = sorted(filelist) try: HDFSBlockReader(multiple_blocks, one_block_per_DN).fetchFilelistBlocksTimed(filelist) - except KeyboardInterrupt, e: + except KeyboardInterrupt as e: printerr("Caught Control-C...", 1) sys.exit(ERRORS["OK"]) - except Exception, e: + except Exception as e: printerr("Error running HDFSBlockReader: %s" % e, 1) if isJavaOOM(e.message): printerr(java_oom_fix_msg()) @@ -206,7 +206,7 @@ class HDFSBlockReader: try: self.client = self.fs.getClient() except: - raise Exception, "Failed to create hdfs client: %s" % get_jython_exception() + raise Exception("Failed to create hdfs client: %s" % get_jython_exception()) #in = client.DFSDataInputStream(self.fh) @@ -214,7 +214,7 @@ class HDFSBlockReader: """ Return the path object for a given filename """ try: path = Path(filename) - except Exception, e: + except Exception as e: return None if path: return path @@ -238,7 +238,7 @@ class HDFSBlockReader: printerr("Failed to get HDFS path object for file " + filename, 1) continue if not self.fs.exists(path): - #raise IOError, "HDFS File not found: %s" % filename + #raise IOError("HDFS File not found: %s" % filename) printerr("HDFS file/dir not found: %s" % filename, 1) continue self.recurse_path(filename, path) @@ -292,7 +292,7 @@ class HDFSBlockReader: if not self.multiple_files: print("Reading File %s\n" % filename) self.fetchFileBlocksTimed(filename, path) - except Exception, e: + except Exception as e: printerr(e, 1) elif self.fs.isDirectory(path): # even if there is only one file under the whole directory tree since it's now different to the specified arg we should print it @@ -308,7 +308,7 @@ class HDFSBlockReader: except: printerr(get_jython_exception().split("\n")[0], 1) else: - raise IOError, ">>> %s is not a file or directory" % filename + raise IOError(">>> %s is not a file or directory" % filename) def fetchFileBlocksTimed(self, filename, path): @@ -323,16 +323,16 @@ class HDFSBlockReader: try: self.fh = self.client.open(filename) except: - raise IOError, "Failed to get client filehandle to HDFS file %s: %s" % (filename, get_jython_exception()) + raise IOError("Failed to get client filehandle to HDFS file %s: %s" % (filename, get_jython_exception())) if self.fh == None: - raise IOError, "Failed to get client filehandle to HDFS file %s" % filename + raise IOError("Failed to get client filehandle to HDFS file %s" % filename) while True: self.block_num += 1 try: block_locations = self.fs.getFileBlockLocations(path, self.offset, self.length) except: - raise IOError, "Failed to get block locations for %sblock %d (offset %d, length %d): %s" % (self.fileoutput, self.block_num, self.offset, self.length, get_jython_exception()) + raise IOError("Failed to get block locations for %sblock %d (offset %d, length %d): %s" % (self.fileoutput, self.block_num, self.offset, self.length, get_jython_exception())) if not block_locations: return for block in block_locations: @@ -344,11 +344,11 @@ class HDFSBlockReader: return self.offset = self.block_offset + self.block_length except: - raise IOError, "Failed to get %sblock %d offset/length: %s" % (self.fileoutput, self.block_num, get_jython_exception()) + raise IOError("Failed to get %sblock %d offset/length: %s" % (self.fileoutput, self.block_num, get_jython_exception())) try: self.readBlockFromDNs() - except Exception, e: - raise IOError, "Failed to read %sblock %d (offset %d, length %d) from datanodes: %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length, e) + except Exception as e: + raise IOError("Failed to read %sblock %d (offset %d, length %d) from datanodes: %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length, e)) def readBlockFromDNs(self): """ Read current block self.offset to self.length from all datanodes """ @@ -375,10 +375,10 @@ class HDFSBlockReader: # self.dn must be populated by here or something is wrong self.dn = self.fh.getCurrentDatanode() if self.dn == None: - raise IOError, "Failed to get current DataNode for %sblock %s offset %s length %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length) + raise IOError("Failed to get current DataNode for %sblock %s offset %s length %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length)) self.host = self.dn.getHostName() if not self.host: - raise Exception, "Failed to get current DataNode host name for %sblock %s offset %s length %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length) + raise Exception("Failed to get current DataNode host name for %sblock %s offset %s length %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length)) # I've come back to the first DN, done with this block if self.host in self.block_read_DNs: return @@ -419,8 +419,8 @@ class HDFSBlockReader: # else: # printerr("ERROR: Failed to read bytes from %sblock %d (offset %d, length %d) [datanode info not available]" % (self.fileoutput, self.block_num, self.block_offset, self.block_length)) # return - except Exception, e: - raise Exception, e + except Exception as e: + raise Exception(e) except: self.dn = self.fh.getCurrentDatanode() if self.dn: @@ -429,7 +429,7 @@ class HDFSBlockReader: self.nodes_failed_reads.add(self.host) print("Error: failed to read %sblock %d (offset %d, length %d) from rack %s datanode %s: %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length, self.rack, self.host, get_jython_exception())) else: - raise IOError, get_jython_exception() + raise IOError(get_jython_exception()) return end_time = time.time() time_taken = end_time - start_time From 64ab8ca72b6f8f33fce57c07ce3f9b1a6cccd755 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 10:26:42 +0100 Subject: [PATCH 0328/2779] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7d5618100..2612b264e 100755 --- a/Makefile +++ b/Makefile @@ -46,8 +46,8 @@ build: $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python - if type apk 2&>/dev/null; then $(MAKE) apk-packages-extra; fi - if type apt-get 2&>/dev/null; then $(MAKE) apt-packages-extra; fi + if type apk 2>/dev/null; then $(MAKE) apk-packages-extra; fi + if type apt-get 2>/dev/null; then $(MAKE) apt-packages-extra; fi $(MAKE) python .PHONY: init From 4336b1986115dd64787d13cd50385f97ce85beda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 10:27:09 +0100 Subject: [PATCH 0329/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0a14375e4..77385e4bb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0a14375e425bfd6b87039ab593849f6998bd65d5 +Subproject commit 77385e4bb136337fddc61596f52e0c13050df0f0 From 19f560b686a8eba4f4f825017a7b8c8deef5cfda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 10:27:09 +0100 Subject: [PATCH 0330/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a1472f403..c654eb304 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a1472f403cc9d16db13147aad62796403eb117d2 +Subproject commit c654eb304cbdfe9f281b0efcc61c786cfd339530 From fc7b3bbef54f8f9093116c9276fadda324b08e75 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 10:49:16 +0100 Subject: [PATCH 0331/2779] updated Makefile --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2612b264e..8cc50ebc9 100755 --- a/Makefile +++ b/Makefile @@ -77,6 +77,9 @@ python: #@bash-tools/python_pip_install.sh snakebite[kerberos] || : @bash-tools/python_pip_install.sh snakebite[kerberos] || : + # Python >= 3.4 - try but accept failure in case we're not on the right version of Python + @if ! python -V | grep -q ^2; then bash-tools/python_pip_install.sh "avro-python3"; fi + # for impyla #$(SUDO_PIP) pip install --upgrade setuptools || : # @@ -103,7 +106,7 @@ python: #if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi # Python >= 2.7 - won't build on 2.6, handle separately and accept failure - $(SUDO_PIP) pip install "ipython[notebook]" || : + @bash-tools/python_pip_install.sh "ipython[notebook]" || : @echo $(MAKE) pycompile @echo From 21fca745b543a424f42a04865cc107bd5e3cb111 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 11:00:59 +0100 Subject: [PATCH 0332/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8cc50ebc9..0ddfa5dbe 100755 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ python: @bash-tools/python_pip_install.sh snakebite[kerberos] || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @if ! python -V | grep -q ^2; then bash-tools/python_pip_install.sh "avro-python3"; fi + @if [ "$(echo "$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi # for impyla #$(SUDO_PIP) pip install --upgrade setuptools || : From b9f183e338bb313111053dcb27708d8b6087def1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 11:06:35 +0100 Subject: [PATCH 0333/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 77385e4bb..f4a432bc0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 77385e4bb136337fddc61596f52e0c13050df0f0 +Subproject commit f4a432bc04585b765263cc6e89d56e1afb20d230 From 177526d7029b8da957511bdf7af585c1614ffc01 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 11:06:35 +0100 Subject: [PATCH 0334/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c654eb304..48b3b3481 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c654eb304cbdfe9f281b0efcc61c786cfd339530 +Subproject commit 48b3b3481a4e97625af0769642b2f6c1cfb7da49 From 2e82d02b95c7e3353fc1fc42347a8607ace55aab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 11:06:53 +0100 Subject: [PATCH 0335/2779] updated requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 257d774f9..59f598b91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ avro>=1.8.1 +# requires Python >= 3.4, build in Makefile instead +#avro-python3>=1.9.0 awscli>=1.16.241 #bitarray>=0.8.1 #cassandra-driver>=3.6.0 From d57997e2f748cd84d49e644fc7dd9f5ed7603e06 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 15:24:53 +0100 Subject: [PATCH 0336/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0ddfa5dbe..1488e4e57 100755 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ python: @bash-tools/python_pip_install.sh snakebite[kerberos] || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @if [ "$(echo "$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi + @if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi # for impyla #$(SUDO_PIP) pip install --upgrade setuptools || : From ba21b6f14f52eb527309698931bc5d58e075042a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Sep 2019 15:28:53 +0100 Subject: [PATCH 0337/2779] update for Python 3 --- validate_avro.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/validate_avro.py b/validate_avro.py index 06d09dfba..f8a896466 100755 --- a/validate_avro.py +++ b/validate_avro.py @@ -35,9 +35,13 @@ import os import re import sys -import avro -from avro.datafile import DataFileReader -from avro.io import DatumReader +# pylint: disable=bare-except +try: + from avro3.datafile import DataFileReader, DataFileException + from avro3.io import DatumReader +except: + from avro.datafile import DataFileReader, DataFileException + from avro.io import DatumReader libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) try: @@ -51,7 +55,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.0' +__version__ = '0.9.1' class AvroValidatorTool(CLI): @@ -86,7 +90,7 @@ def check_avro(self, filehandle): try: DataFileReader(filehandle, DatumReader()) print(self.valid_avro_msg) - except avro.datafile.DataFileException as _: + except DataFileException as _: if 'snappy' in str(_): die("%s => ERROR: %s - Is the python-snappy module installed? ('pip install python-snappy')" \ % (filehandle.name, _)) From b4e0cb995eb4fa11dfcefb379f06691109ed2579 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 10:30:51 +0100 Subject: [PATCH 0338/2779] added find_python_library_path.py --- find_python_library_path.py | 73 +++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 find_python_library_path.py diff --git a/find_python_library_path.py b/find_python_library_path.py new file mode 100755 index 000000000..5fe8817f2 --- /dev/null +++ b/find_python_library_path.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-09-27 +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Simply tool to print the local path to one or more libraries given as arguments + +Useful for finding where things are installed on different operating systems like Mac vs Linux + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) + + +def usage(): + print('usage: {} [ ...]'.format(os.path.basename(sys.argv[0])), file=sys.stderr) + sys.exit(3) + + +def main(): + exitcode = 0 + for module in sys.argv[1:]: + try: + # pylint: disable=wrong-import-position,import-error + imported = __import__(module) + # sys doesn't have __file__ but it does have base_prefix + attribs = dir(imported) + potential_attribs = ['__file__', 'base_prefix', 'base_exec_prefix', 'exec_prefix'] + found = False + for _ in potential_attribs: + if _ in attribs: + print(getattr(imported, _)) + found = True + break + if not found: + raise AttributeError('could not find any attribute to determine location out of: {}'\ + .format(', '.join(potential_attribs))) + except AttributeError as _: + print('attributes not found: {} - {}'.format(module, _), file=sys.stderr) + exitcode = max(1, exitcode) + except ImportError as _: + print('module not found: {} - {}'.format(module, _), file=sys.stderr) + exitcode = max(2, exitcode) + sys.exit(exitcode) + + +if __name__ == '__main__': + main() From 66746cff011f0cc71fd11a2ce7070554703d5410 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 10:32:08 +0100 Subject: [PATCH 0339/2779] updated tests/test_welcome.sh --- tests/test_welcome.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_welcome.sh b/tests/test_welcome.sh index 44270e6a1..fb1998284 100755 --- a/tests/test_welcome.sh +++ b/tests/test_welcome.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh -./welcome.py $@ +./welcome.py "$@" From 04bffbbbc18c2a068c0e89e4aa98e2787f6e46fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 11:03:20 +0100 Subject: [PATCH 0340/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f4a432bc0..f0367e29e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f4a432bc04585b765263cc6e89d56e1afb20d230 +Subproject commit f0367e29eb36aadad92e108f3c1ae05302880429 From da228898d3bb621d4c6fe04b1bdd0fa0061b4bcc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 11:29:35 +0100 Subject: [PATCH 0341/2779] added test_find_python_library_path.sh --- tests/test_find_python_library_path.sh | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 tests/test_find_python_library_path.sh diff --git a/tests/test_find_python_library_path.sh b/tests/test_find_python_library_path.sh new file mode 100755 index 000000000..36bdd2261 --- /dev/null +++ b/tests/test_find_python_library_path.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -eu +[ -n "${DEBUG:-}" ] && set -x +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd "$srcdir/.."; + +# shellcheck disable=SC1091 +. ./tests/utils.sh + +run_grep '/os.py[co]?$' ./find_python_library_path.py os +run_grep '/logging/__init__.py[co]?$' ./find_python_library_path.py logging +ERRCODE=2 run_grep '' ./find_python_library_path.py nonexistentmodule +run_grep 'python' ./find_python_library_path.py sys From 657c2999728bfacffae182f3bc85e318e582e3a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 12:01:32 +0100 Subject: [PATCH 0342/2779] updated find_python_library_path.py --- find_python_library_path.py | 47 +++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/find_python_library_path.py b/find_python_library_path.py index 5fe8817f2..85a32734d 100755 --- a/find_python_library_path.py +++ b/find_python_library_path.py @@ -20,6 +20,8 @@ Useful for finding where things are installed on different operating systems like Mac vs Linux +Tested on Python 2.7 and Python 3.x on Mac and Linux + """ from __future__ import absolute_import @@ -41,27 +43,42 @@ def usage(): print('usage: {} [ ...]'.format(os.path.basename(sys.argv[0])), file=sys.stderr) sys.exit(3) +def get_module_location(module): + # pylint: disable=wrong-import-position,import-error + imported_module = __import__(module) + # sys doesn't have __file__ but it does have base_prefix + attribs = dir(imported_module) + # - looks like 'sys.base_prefix' / 'sys.base_exec_prefix' are only available in IPython + # - don't use 'executable' as it will point to a generic location eg. /usr/bin/python + # or /usr/local/opt/python/bin/python3.7) instead of the actual library path + # like /usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7 + #potential_attribs = ['__file__', 'exec_prefix', 'base_prefix', 'base_exec_prefix'] + potential_attribs = ['__file__', 'exec_prefix'] #, 'executable'] + for potential_attrib in potential_attribs: + if potential_attrib in attribs: + value = getattr(imported_module, potential_attrib) + if potential_attrib == 'exec_prefix': + if not 'Python.framework' in value: + continue + return value + # to account for sys not having __file__ attribute + if '__name__' in attribs and imported_module.__name__ == 'sys': + python_site_packages = 'python{}.{}/site-packages'\ + .format(sys.version_info.major, sys.version_info.minor) + for path in sys.path: + if path.endswith(python_site_packages): + value = path.rsplit('/site-packages')[0] + return value + raise AttributeError('could not find any attribute to determine location out of: {}'\ + .format(', '.join(potential_attribs))) def main(): exitcode = 0 for module in sys.argv[1:]: try: - # pylint: disable=wrong-import-position,import-error - imported = __import__(module) - # sys doesn't have __file__ but it does have base_prefix - attribs = dir(imported) - potential_attribs = ['__file__', 'base_prefix', 'base_exec_prefix', 'exec_prefix'] - found = False - for _ in potential_attribs: - if _ in attribs: - print(getattr(imported, _)) - found = True - break - if not found: - raise AttributeError('could not find any attribute to determine location out of: {}'\ - .format(', '.join(potential_attribs))) + print(get_module_location(module)) except AttributeError as _: - print('attributes not found: {} - {}'.format(module, _), file=sys.stderr) + print('attributes not found in module: {} - {}'.format(module, _), file=sys.stderr) exitcode = max(1, exitcode) except ImportError as _: print('module not found: {} - {}'.format(module, _), file=sys.stderr) From e3636af4dcf3e2e44b9f559eb3a0f28b63a0e1b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 12:01:46 +0100 Subject: [PATCH 0343/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f0367e29e..d4e4c825e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f0367e29eb36aadad92e108f3c1ae05302880429 +Subproject commit d4e4c825e5639c040c8708060f0e9667f2d9b8ee From 17492815397a762529d69326c50512515879a351 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 12:01:51 +0100 Subject: [PATCH 0344/2779] updated tests/test_welcome.sh --- tests/test_welcome.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_welcome.sh b/tests/test_welcome.sh index fb1998284..1be838108 100755 --- a/tests/test_welcome.sh +++ b/tests/test_welcome.sh @@ -20,6 +20,6 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; # shellcheck disable=SC1091 -. ./tests/utils.sh +#. ./tests/utils.sh ./welcome.py "$@" From 416cb2159999043c58a8fc115a817e474d60c549 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 12:38:21 +0100 Subject: [PATCH 0345/2779] updated tests/test_find_python_library_path.sh --- tests/test_find_python_library_path.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_find_python_library_path.sh b/tests/test_find_python_library_path.sh index 36bdd2261..3d60b7c47 100755 --- a/tests/test_find_python_library_path.sh +++ b/tests/test_find_python_library_path.sh @@ -26,3 +26,5 @@ run_grep '/os.py[co]?$' ./find_python_library_path.py os run_grep '/logging/__init__.py[co]?$' ./find_python_library_path.py logging ERRCODE=2 run_grep '' ./find_python_library_path.py nonexistentmodule run_grep 'python' ./find_python_library_path.py sys +ERRCODE=3 run_grep '' ./find_python_library_path.py +run_usage ./find_python_library_path.py From 9308ec20b4ae22e53f2e5e432c68114285941333 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 12:38:55 +0100 Subject: [PATCH 0346/2779] updated find_python_library_path.py --- find_python_library_path.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/find_python_library_path.py b/find_python_library_path.py index 85a32734d..e1ff0f269 100755 --- a/find_python_library_path.py +++ b/find_python_library_path.py @@ -16,7 +16,7 @@ """ -Simply tool to print the local path to one or more libraries given as arguments +Simple tool to print the local path to one or more libraries given as arguments Useful for finding where things are installed on different operating systems like Mac vs Linux @@ -33,14 +33,16 @@ import sys __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) def usage(): - print('usage: {} [ ...]'.format(os.path.basename(sys.argv[0])), file=sys.stderr) + print(__doc__.strip()) + print('\n\nusage: {} [ ...]\n'\ + .format(os.path.basename(sys.argv[0])), file=sys.stderr) sys.exit(3) def get_module_location(module): @@ -74,6 +76,11 @@ def get_module_location(module): def main(): exitcode = 0 + if len(sys.argv) < 2: + usage() + for arg in sys.argv: + if arg.startswith('-'): + usage() for module in sys.argv[1:]: try: print(get_module_location(module)) From 662d95e5c63674a0602ada63410d2fff1450e9a0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 12:44:52 +0100 Subject: [PATCH 0347/2779] updated tests/test_find_python_library_path.sh --- tests/test_find_python_library_path.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_find_python_library_path.sh b/tests/test_find_python_library_path.sh index 3d60b7c47..1d52e977d 100755 --- a/tests/test_find_python_library_path.sh +++ b/tests/test_find_python_library_path.sh @@ -2,7 +2,7 @@ # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon -# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) +# Date: 2019-09-27 # # https://github.com/harisekhon/devops-python-tools # From f048fd4efcd87304dbb230dfcb288d969d13e996 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 13:04:55 +0100 Subject: [PATCH 0348/2779] updated find_python_library_path.py --- find_python_library_path.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/find_python_library_path.py b/find_python_library_path.py index e1ff0f269..ab9ad50f4 100755 --- a/find_python_library_path.py +++ b/find_python_library_path.py @@ -20,6 +20,8 @@ Useful for finding where things are installed on different operating systems like Mac vs Linux +If no libraries are specified, finds the default library location as if the 'sys' argument was given + Tested on Python 2.7 and Python 3.x on Mac and Linux """ @@ -33,7 +35,7 @@ import sys __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) @@ -77,7 +79,7 @@ def get_module_location(module): def main(): exitcode = 0 if len(sys.argv) < 2: - usage() + sys.argv.append('sys') for arg in sys.argv: if arg.startswith('-'): usage() From 8c1b5169dd44af0753ae9ead74842ea150a2c821 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 13:05:36 +0100 Subject: [PATCH 0349/2779] updated tests/test_find_python_library_path.sh --- tests/test_find_python_library_path.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_find_python_library_path.sh b/tests/test_find_python_library_path.sh index 1d52e977d..b8afab795 100755 --- a/tests/test_find_python_library_path.sh +++ b/tests/test_find_python_library_path.sh @@ -26,5 +26,5 @@ run_grep '/os.py[co]?$' ./find_python_library_path.py os run_grep '/logging/__init__.py[co]?$' ./find_python_library_path.py logging ERRCODE=2 run_grep '' ./find_python_library_path.py nonexistentmodule run_grep 'python' ./find_python_library_path.py sys -ERRCODE=3 run_grep '' ./find_python_library_path.py +run_grep 'python' ./find_python_library_path.py run_usage ./find_python_library_path.py From 7d7eebb772b30c1a2702c432685a800f850a0446 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 13:17:56 +0100 Subject: [PATCH 0350/2779] updated tests/test_find_python_library_path.sh --- tests/test_find_python_library_path.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_find_python_library_path.sh b/tests/test_find_python_library_path.sh index b8afab795..cb159a79f 100755 --- a/tests/test_find_python_library_path.sh +++ b/tests/test_find_python_library_path.sh @@ -27,4 +27,5 @@ run_grep '/logging/__init__.py[co]?$' ./find_python_library_path.py logging ERRCODE=2 run_grep '' ./find_python_library_path.py nonexistentmodule run_grep 'python' ./find_python_library_path.py sys run_grep 'python' ./find_python_library_path.py -run_usage ./find_python_library_path.py +run_usage ./find_python_library_path.py - +run_usage ./find_python_library_path.py --help From 9846febe470fc1edf40117b3fedad0ba9a510f50 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 13:23:29 +0100 Subject: [PATCH 0351/2779] added pythonpath.py --- pythonpath.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 pythonpath.py diff --git a/pythonpath.py b/pythonpath.py new file mode 100755 index 000000000..2d9d8f4d6 --- /dev/null +++ b/pythonpath.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-09-27 +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Simple tool to print the Python sys.path, one per line + +Tested on Python 2.7 and Python 3.x on Mac and Linux + +""" + +from __future__ import print_function +#from __future__ import unicode_literals + +import sys + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +#libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +#sys.path.append(libdir) + + +def main(): + for path in sys.path: + if path: + print(path) + + +if __name__ == '__main__': + main() From b3f5ccdd8431ceaf089901a05635917d3ea8e4b9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 21:18:00 +0100 Subject: [PATCH 0352/2779] updated find_python_library_path.py --- find_python_library_path.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/find_python_library_path.py b/find_python_library_path.py index ab9ad50f4..24d0aab2a 100755 --- a/find_python_library_path.py +++ b/find_python_library_path.py @@ -31,6 +31,7 @@ from __future__ import print_function #from __future__ import unicode_literals +import getpass import os import sys @@ -62,16 +63,19 @@ def get_module_location(module): if potential_attrib in attribs: value = getattr(imported_module, potential_attrib) if potential_attrib == 'exec_prefix': - if not 'Python.framework' in value: + if not 'Python.framework' in value and not 'site-packages' in value: continue return value # to account for sys not having __file__ attribute if '__name__' in attribs and imported_module.__name__ == 'sys': - python_site_packages = 'python{}.{}/site-packages'\ - .format(sys.version_info.major, sys.version_info.minor) + user = getpass.getuser() for path in sys.path: - if path.endswith(python_site_packages): - value = path.rsplit('/site-packages')[0] + # don't return local /Users/$USER/Library/Python/2.7/lib/python/site-packages + # as that is not the source of sys + if user in path: + continue + if path.endswith('/site-packages'): + #value = path.rsplit('/site-packages')[0] return value raise AttributeError('could not find any attribute to determine location out of: {}'\ .format(', '.join(potential_attribs))) From b2ac50ef4f1a91e33655b4544d39b5fe257dd464 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 22:24:39 +0100 Subject: [PATCH 0353/2779] renamed find_python_library_path.py to python_find_library_path.py --- find_python_library_path.py => python_find_library_path.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename find_python_library_path.py => python_find_library_path.py (100%) diff --git a/find_python_library_path.py b/python_find_library_path.py similarity index 100% rename from find_python_library_path.py rename to python_find_library_path.py From cacc28b0f8115ea39b698800e91f312b6d215e43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Sep 2019 23:22:04 +0100 Subject: [PATCH 0354/2779] updated tests/test_find_python_library_path.sh --- tests/test_find_python_library_path.sh | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/test_find_python_library_path.sh b/tests/test_find_python_library_path.sh index cb159a79f..6a13f5779 100755 --- a/tests/test_find_python_library_path.sh +++ b/tests/test_find_python_library_path.sh @@ -22,10 +22,22 @@ cd "$srcdir/.."; # shellcheck disable=SC1091 . ./tests/utils.sh -run_grep '/os.py[co]?$' ./find_python_library_path.py os -run_grep '/logging/__init__.py[co]?$' ./find_python_library_path.py logging -ERRCODE=2 run_grep '' ./find_python_library_path.py nonexistentmodule -run_grep 'python' ./find_python_library_path.py sys -run_grep 'python' ./find_python_library_path.py -run_usage ./find_python_library_path.py - -run_usage ./find_python_library_path.py --help +section "Python Find Library Path" + +start_time="$(start_timer "python_find_library_path")" + +run ./pythonpath.py + +run_grep '/os.py[co]?$' ./python_find_library_path.py os +run_grep '/logging/__init__.py[co]?$' ./python_find_library_path.py logging +ERRCODE=2 run_grep '' ./python_find_library_path.py nonexistentmodule +run_grep 'python' ./python_find_library_path.py sys +run_grep 'python' ./python_find_library_path.py +run_usage ./python_find_library_path.py - +run_usage ./python_find_library_path.py --help + +echo +# shellcheck disable=SC2154 +echo "Total Tests run: $run_count" +time_taken "$start_time" "SUCCESS! All tests for python_find_library_path.py completed in" +echo From 8d16971b9c8c6038ccf79d72375e786e0537dd4f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Sep 2019 18:59:30 +0100 Subject: [PATCH 0355/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d4e4c825e..27393b799 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d4e4c825e5639c040c8708060f0e9667f2d9b8ee +Subproject commit 27393b79972ca51cdb61fb0692e7379a57b389c9 From da069beff17cd6292551b17d0e6c73f51b37a8dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Sep 2019 18:59:30 +0100 Subject: [PATCH 0356/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 48b3b3481..049f55858 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 48b3b3481a4e97625af0769642b2f6c1cfb7da49 +Subproject commit 049f558580b50c0e93d20801b20d9f555514dcd5 From a2cb9e3a569e7711ad3f4ef8477bd2af0218747e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Sep 2019 13:22:22 +0100 Subject: [PATCH 0357/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 27393b799..9dd42287a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 27393b79972ca51cdb61fb0692e7379a57b389c9 +Subproject commit 9dd42287a4ddca1c02aa8bb82bf3784513467266 From f90a4d6f9e825637abd132d05e0005009bba246d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Sep 2019 13:22:22 +0100 Subject: [PATCH 0358/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 049f55858..776dc47a5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 049f558580b50c0e93d20801b20d9f555514dcd5 +Subproject commit 776dc47a5be1747b5457c585b3a50c3c6c5a7d32 From 0596fb5aa0a6dfd0e562e78114001a1a363a14ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 00:30:21 +0100 Subject: [PATCH 0359/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 27393b799..7d185696f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 27393b79972ca51cdb61fb0692e7379a57b389c9 +Subproject commit 7d185696f55d312ae7e4c07664fcc3dc1791738e From 282ed653fe496631416a0b9b66e2f0e661486705 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 00:30:21 +0100 Subject: [PATCH 0360/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 049f55858..4bc93b381 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 049f558580b50c0e93d20801b20d9f555514dcd5 +Subproject commit 4bc93b381f13a5408a9364f1829bd8e483a23f48 From 7aa94d5cfe91de36671a35254cea8efa0e1d11a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 12:08:58 +0100 Subject: [PATCH 0361/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4bc93b381..d4043ff34 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4bc93b381f13a5408a9364f1829bd8e483a23f48 +Subproject commit d4043ff346c84b0ffe39d25d1fc6f4b680fb4f98 From 4927fbf02fa8ab1ffb335e91eea178f76936c6ce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:17:36 +0100 Subject: [PATCH 0362/2779] updated tests/utils.sh --- tests/utils.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/utils.sh b/tests/utils.sh index 01cd59ac9..57c32ca46 100755 --- a/tests/utils.sh +++ b/tests/utils.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC1090 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -28,11 +29,13 @@ export COMPOSE_PROJECT_NAME="pytools" if [ -n "${TRAVIS:-}" ]; then sudo=sudo else + # will be used in sourcing scripts + # shellcheck disable=SC2034 sudo="" fi set +o pipefail -spark_home="$(ls -d tests/spark-*-bin-hadoop* 2>/dev/null | head -n 1)" +spark_home="$(find . -type d -path 'tests/spark-*-bin-hadoop*' 2>/dev/null | head -n 1)" set -o pipefail if [ -n "$spark_home" ]; then export SPARK_HOME="$spark_home" @@ -56,7 +59,7 @@ sample_files(){ "$data_dir/../../README.md" \ ; do local excluded=0 - for ext in $@; do + for ext in "$@"; do if [ "${filename##*.}" = "$ext" ]; then excluded=1 break @@ -69,9 +72,9 @@ sample_files(){ } check_broken_sample_files(){ - echo "Now checking non $@ files to detect successful failure:" + echo "Now checking non $* files to detect successful failure:" echo - for filename in $(sample_files $@); do + for filename in $(sample_files "$@"); do check_broken "$filename" done } From fa7efada4c6f781f656901c5c0a736cbbf11a815 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:18:35 +0100 Subject: [PATCH 0363/2779] updated tests/test_yamllint.sh --- tests/test_yamllint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_yamllint.sh b/tests/test_yamllint.sh index 411432530..31d808ff6 100755 --- a/tests/test_yamllint.sh +++ b/tests/test_yamllint.sh @@ -21,6 +21,7 @@ cfg="$srcdir/../.yamllint" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Running Yaml lint" From 75582b023557c2e5f79bc7819a234e74a2c0615a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:21:23 +0100 Subject: [PATCH 0364/2779] updated ambari_ams_metrics.sh --- ambari_ams_metrics.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ambari_ams_metrics.sh b/ambari_ams_metrics.sh index 0c9e92a65..ef84defea 100755 --- a/ambari_ams_metrics.sh +++ b/ambari_ams_metrics.sh @@ -98,13 +98,14 @@ if [ -z "$ams_host" ]; then elif [ -z "$ams_port" ]; then usage "--port not defined" fi -if [ -n "$node_list" -a -z "$metric" ]; then +if [ -n "$node_list" ] && [ -z "$metric" ]; then usage "--nodes option requires --metric. See --list-metrics for a list of available metrics" fi check_bin(){ local bin="$1" - if ! which $bin &>/dev/null; then + # shellcheck disable=SC2230 + if ! which "$bin" &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi @@ -125,7 +126,7 @@ list_metrics(){ list_hosts(){ curl -s "$ams_host:$ams_port/ws/v1/timeline/metrics/hosts" | python -m json.tool | - egrep '^[[:space:]]*".+":[[:space:]]* \[[[:space:]]*$' | + grep -E '^[[:space:]]*".+":[[:space:]]* \[[[:space:]]*$' | sed 's/"//g;s/:.*//;s/[[:space:]]*//g' | grep -v fakehostname | sort || From a2414f940e23067de3aeec2b9281d6edca278072 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:21:38 +0100 Subject: [PATCH 0365/2779] updated ambari_cancel_all_requests.sh --- ambari_cancel_all_requests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ambari_cancel_all_requests.sh b/ambari_cancel_all_requests.sh index 460e2b76f..15e079525 100755 --- a/ambari_cancel_all_requests.sh +++ b/ambari_cancel_all_requests.sh @@ -54,7 +54,7 @@ echo "querying Ambari for request IDs" curl -u "$AMBARI_USER:$AMBARI_PASSWORD" "$PROTOCOL://$AMBARI_HOST:$AMBARI_PORT/api/v1/clusters/$AMBARI_CLUSTER/requests" | grep id | awk '{print $3}' | -while read id; do +while read -r id; do echo "requesting cancellation of request $id" # Unfortunately this call returns 200 OK even if your user account doesn't have permission to do this operation, but will fail on the requests query above for wrong username/password curl -u "$AMBARI_USER:$AMBARI_PASSWORD" -i -H "X-Requested-By: $AMBARI_USER ($USER)" -X PUT -d '{"Requests":{"request_status":"ABORTED","abort_reason":"Aborted by user"}}' "$PROTOCOL://$AMBARI_HOST:$AMBARI_PORT/api/v1/clusters/$AMBARI_CLUSTER/requests/$id" From b296848040d9eb35bf67e1d15779b975a6368af8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:22:24 +0100 Subject: [PATCH 0366/2779] updated anonymize_parallel.sh --- anonymize_parallel.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 5b8a1fb4c..8799955fa 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -17,11 +17,13 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# shellcheck disable=SC1090 . "$srcdir/bash-tools/lib/utils.sh" # re-establish srcdir local to this script since util.sh include brings its own srcdir srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# shellcheck disable=SC2120 usage(){ if [ -n "$*" ]; then echo "$@" >&2 From 07db6eb81d6acd80e73a1004f462f2d4464f06f7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:22:42 +0100 Subject: [PATCH 0367/2779] updated docker_pull_all_images.sh --- docker_pull_all_images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_pull_all_images.sh b/docker_pull_all_images.sh index 44fd78f66..430ac70d5 100755 --- a/docker_pull_all_images.sh +++ b/docker_pull_all_images.sh @@ -16,7 +16,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(dirname $0)" +srcdir="$(dirname "$0")" for docker_image in $("$srcdir/dockerhub_search.py" -q harisekhon); do echo "docker pull $docker_image" From 1b86875615450e67e8da3aff10ed61c9444ea28e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:22:54 +0100 Subject: [PATCH 0368/2779] updated docker_pull_all_images_all_tags.sh --- docker_pull_all_images_all_tags.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_pull_all_images_all_tags.sh b/docker_pull_all_images_all_tags.sh index 5276ff6dd..3656395cb 100755 --- a/docker_pull_all_images_all_tags.sh +++ b/docker_pull_all_images_all_tags.sh @@ -16,7 +16,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(dirname $0)" +srcdir="$(dirname "$0")" for docker_image in $("$srcdir/dockerhub_search.py" -q harisekhon | grep -v 'presto.*dev'); do echo "pulling all tags for docker image $docker_image:" From f549526c09154242c204e868a1d570c730ad5d95 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:23:12 +0100 Subject: [PATCH 0369/2779] updated docker_pull_all_tags.sh --- docker_pull_all_tags.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_pull_all_tags.sh b/docker_pull_all_tags.sh index ade394ef3..e555af230 100755 --- a/docker_pull_all_tags.sh +++ b/docker_pull_all_tags.sh @@ -16,7 +16,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(dirname $0)" +srcdir="$(dirname "$0")" docker_image="${1:-}" From d40800ead55e3cd6b63f6e49787e5233a491863c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:23:49 +0100 Subject: [PATCH 0370/2779] updated hbase_scan_table_column_names.sh --- hbase_scan_table_column_names.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase_scan_table_column_names.sh b/hbase_scan_table_column_names.sh index 08e6ab5a1..f22cd63f5 100755 --- a/hbase_scan_table_column_names.sh +++ b/hbase_scan_table_column_names.sh @@ -17,7 +17,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x usage(){ - [ -n "${@:-}" ] && echo "$@" + [ -n "${*:-}" ] && echo "$@" echo echo "Fast and dirty script to scan an HBase table to determine all columns (column family + qualifiers) @@ -45,6 +45,7 @@ if [ -z "${table:-}" ]; then usage "ERROR: Table not defined, use --table switch" fi +# shellcheck disable=SC2230 if ! which hbase &>/dev/null; then usage "ERROR: hbase command not found in \$PATH ($PATH)" fi From b58f0029c3a5adfda858694207618be2af8ad0c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:25:32 +0100 Subject: [PATCH 0371/2779] updated hbase_table_regions_by_regionserver.sh --- hbase_table_regions_by_regionserver.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hbase_table_regions_by_regionserver.sh b/hbase_table_regions_by_regionserver.sh index 6ba8244ff..588972f47 100755 --- a/hbase_table_regions_by_regionserver.sh +++ b/hbase_table_regions_by_regionserver.sh @@ -18,7 +18,6 @@ set -euo pipefail regionservers="" port="${HBASE_REGIONSERVER_PORT:-16030}" -metrics="metrics" usage(){ if [ -n "$*" ]; then @@ -66,7 +65,8 @@ fi check_bin(){ local bin="$1" - if ! which $bin &>/dev/null; then + # shellcheck disable=SC2230 + if ! which "$bin" &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi @@ -79,6 +79,7 @@ if [ -z "${DEBUG:-}" ]; then fi for regionserver in $(tr ' ' '\n' <<< "$regionservers" | sort -u); do echo "$regionserver:" + # shellcheck disable=SC2086 curl $curl_options "$regionserver:$port/jmx" | grep "_table_.*${table}.*_region_" | sed 's/.*_table_//; s/_region_/ /; s/_.*$//' | From 1a29ffe9fe5a4efe55f3c2367731b4d9fa2840b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:26:02 +0100 Subject: [PATCH 0372/2779] updated hbase_table_regionserver_distribution.sh --- hbase_table_regionserver_distribution.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hbase_table_regionserver_distribution.sh b/hbase_table_regionserver_distribution.sh index e7f5c69cd..252a9a3d4 100755 --- a/hbase_table_regionserver_distribution.sh +++ b/hbase_table_regionserver_distribution.sh @@ -24,7 +24,6 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" usage(){ echo "usage: $0 " @@ -37,9 +36,9 @@ if [ $# -lt 1 ]; then usage fi -for server in $@; do +for server in "$@"; do echo "$server:" - curl -s $server:$port/jmx | + curl -s "$server:$port/jmx" | grep 'Namespace_.*_metric_getCount"[[:space:]]*:[[:space:]]*' | sed 's/.*_table_//; s/_region_.*//' | sort | From 4e6545be7dfe8dc1def587b8c60350a61107ccf1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:26:29 +0100 Subject: [PATCH 0373/2779] updated hbase_uncompacted_regions.sh --- hbase_uncompacted_regions.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hbase_uncompacted_regions.sh b/hbase_uncompacted_regions.sh index e4afcd96a..7b092c06d 100755 --- a/hbase_uncompacted_regions.sh +++ b/hbase_uncompacted_regions.sh @@ -19,7 +19,6 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" usage(){ echo "usage: $0 " @@ -32,8 +31,8 @@ if [ $# -lt 1 ]; then usage fi -for server in $@; do - curl -s $server:$port/jmx | +for server in "$@"; do + curl -s "$server:$port/jmx" | grep 'compactionsCompletedCount"[[:space:]]*:[[:space:]]*0,$' | sed 's/.*_table_//; s/_region_/ /; s/_metric_.*//' | column -t From 471e3d8706647b408e9fe54a48dcab95462cc9cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Oct 2019 16:26:51 +0100 Subject: [PATCH 0374/2779] updated jython_install.sh --- jython_install.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jython_install.sh b/jython_install.sh index 690b0421a..b56c94cac 100755 --- a/jython_install.sh +++ b/jython_install.sh @@ -15,7 +15,6 @@ set -eu [ -n "${DEBUG:-}" ] && set -x -srcdir="$(cd "$(dirname "$0")" && pwd)" JYTHON_VERSION="${1:-2.7.0}" @@ -23,7 +22,7 @@ JYTHON_INSTALL_DIR="${2:-/opt/jython-$JYTHON_VERSION}" # not set in busybox #[ $EUID -eq 0 ] && sudo="" || sudo=sudo -[ $(whoami) = "root" ] && sudo="" || sudo=sudo +[ "$(whoami)" = "root" ] && sudo="" || sudo=sudo # installer will tell us if dir isn't empty #if ! [ -e "$JYTHON_INSTALL_DIR" ]; then From 83cf362be325d0cf758ceec3ec274a224b4b5f76 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 4 Oct 2019 11:24:57 +0100 Subject: [PATCH 0375/2779] replaced which with type -P --- ambari_ams_metrics.sh | 2 +- hbase_scan_table_column_names.sh | 3 +-- hbase_table_regions_by_regionserver.sh | 2 +- opentsdb_list_metrics.sh | 2 +- opentsdb_list_metrics_hbase.sh | 2 +- tests/excluded.sh | 2 +- tests/syntax.sh | 4 ++-- tests/test_dockerfiles_check_git_branches.sh | 2 +- tests/test_dockerfiles_check_git_tags.sh | 2 +- tests/test_git_check_branches_upstream.sh | 2 +- tests/test_spark_csv_to_avro.sh | 2 +- tests/test_spark_csv_to_parquet.sh | 2 +- tests/test_spark_json_to_avro.sh | 2 +- tests/test_spark_json_to_parquet.sh | 2 +- tests/test_spark_z_avro_to_parquet.sh | 2 +- tests/test_spark_z_parquet_to_avro.sh | 2 +- tests/test_validate_multimedia.sh | 6 +++--- 17 files changed, 20 insertions(+), 21 deletions(-) diff --git a/ambari_ams_metrics.sh b/ambari_ams_metrics.sh index ef84defea..85f24ee53 100755 --- a/ambari_ams_metrics.sh +++ b/ambari_ams_metrics.sh @@ -105,7 +105,7 @@ fi check_bin(){ local bin="$1" # shellcheck disable=SC2230 - if ! which "$bin" &>/dev/null; then + if ! type -P "$bin" &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi diff --git a/hbase_scan_table_column_names.sh b/hbase_scan_table_column_names.sh index f22cd63f5..f2d62ad44 100755 --- a/hbase_scan_table_column_names.sh +++ b/hbase_scan_table_column_names.sh @@ -45,8 +45,7 @@ if [ -z "${table:-}" ]; then usage "ERROR: Table not defined, use --table switch" fi -# shellcheck disable=SC2230 -if ! which hbase &>/dev/null; then +if ! type -P hbase &>/dev/null; then usage "ERROR: hbase command not found in \$PATH ($PATH)" fi diff --git a/hbase_table_regions_by_regionserver.sh b/hbase_table_regions_by_regionserver.sh index 588972f47..5e603e289 100755 --- a/hbase_table_regions_by_regionserver.sh +++ b/hbase_table_regions_by_regionserver.sh @@ -66,7 +66,7 @@ fi check_bin(){ local bin="$1" # shellcheck disable=SC2230 - if ! which "$bin" &>/dev/null; then + if ! type -P "$bin" &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi diff --git a/opentsdb_list_metrics.sh b/opentsdb_list_metrics.sh index b82c66113..3a2675738 100755 --- a/opentsdb_list_metrics.sh +++ b/opentsdb_list_metrics.sh @@ -72,7 +72,7 @@ done check_bin(){ local bin="$1" - if ! which $bin &>/dev/null; then + if ! type -P $bin &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi diff --git a/opentsdb_list_metrics_hbase.sh b/opentsdb_list_metrics_hbase.sh index 4b2781d70..56aaddfa9 100755 --- a/opentsdb_list_metrics_hbase.sh +++ b/opentsdb_list_metrics_hbase.sh @@ -73,7 +73,7 @@ done check_bin(){ local bin="$1" - if ! which $bin &>/dev/null; then + if ! type -P $bin &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi diff --git a/tests/excluded.sh b/tests/excluded.sh index 7bbe7c34e..b02a8c6bb 100755 --- a/tests/excluded.sh +++ b/tests/excluded.sh @@ -31,7 +31,7 @@ isExcluded(){ if is_inside_docker && [[ "$prog" =~ spark_* ]]; then return 0 fi - if which git &>/dev/null; then + if type -P git &>/dev/null; then commit="$(git log "$prog" | head -n1 | grep 'commit')" if [ -z "$commit" ]; then return 0 diff --git a/tests/syntax.sh b/tests/syntax.sh index 8008e1fbd..26a8f16c8 100755 --- a/tests/syntax.sh +++ b/tests/syntax.sh @@ -23,13 +23,13 @@ cd "$srcdir/.."; for x in $(echo *.py *.jy 2>/dev/null); do isExcluded "$x" && continue - if which flake8 &> /dev/null; then + if type -P flake8 &> /dev/null; then echo "flake8 $x" flake8 --max-line-length=120 --statistics $x echo; hr; echo fi for y in pyflakes pychecker; do - if which $y &>/dev/null; then + if type -P $y &>/dev/null; then echo "$y $x" $y $x echo; hr; echo diff --git a/tests/test_dockerfiles_check_git_branches.sh b/tests/test_dockerfiles_check_git_branches.sh index 90bf1ac7a..ec2be2a11 100755 --- a/tests/test_dockerfiles_check_git_branches.sh +++ b/tests/test_dockerfiles_check_git_branches.sh @@ -25,7 +25,7 @@ cd "$srcdir/.." section "Testing Dockerfiles Check Git branches" -if which git &>dev/null; then +if type -P git &>dev/null; then if ! [ -d Dockerfiles ]; then git clone https://github.com/harisekhon/Dockerfiles else diff --git a/tests/test_dockerfiles_check_git_tags.sh b/tests/test_dockerfiles_check_git_tags.sh index 372076138..33556738c 100755 --- a/tests/test_dockerfiles_check_git_tags.sh +++ b/tests/test_dockerfiles_check_git_tags.sh @@ -25,7 +25,7 @@ cd "$srcdir/.." section "Testing Dockerfiles Check Git Tags" -if which git &>dev/null; then +if type -P git &>dev/null; then if ! [ -d Dockerfiles ]; then git clone https://github.com/harisekhon/Dockerfiles else diff --git a/tests/test_git_check_branches_upstream.sh b/tests/test_git_check_branches_upstream.sh index 8ae7a55ae..878529a01 100755 --- a/tests/test_git_check_branches_upstream.sh +++ b/tests/test_git_check_branches_upstream.sh @@ -25,7 +25,7 @@ cd "$srcdir/.." section "Testing Git check branches upstream" -if which git &>/dev/null; then +if type -P git &>/dev/null; then if ! [ -d Dockerfiles ]; then git clone https://github.com/harisekhon/Dockerfiles else diff --git a/tests/test_spark_csv_to_avro.sh b/tests/test_spark_csv_to_avro.sh index c6fed4a38..05cf1bc17 100755 --- a/tests/test_spark_csv_to_avro.sh +++ b/tests/test_spark_csv_to_avro.sh @@ -41,7 +41,7 @@ for SPARK_VERSION in $SPARK_VERSIONS; do if ! [ -f "$tar" ]; then echo "fetching $tar" # some systems don't have wget - if which wget &>/dev/null; then + if type -P wget &>/dev/null; then wget "http://d3kbcqa49mib13.cloudfront.net/$tar" else curl -L "http://d3kbcqa49mib13.cloudfront.net/$tar" > "$tar" diff --git a/tests/test_spark_csv_to_parquet.sh b/tests/test_spark_csv_to_parquet.sh index 4575382d6..1fc4b6b55 100755 --- a/tests/test_spark_csv_to_parquet.sh +++ b/tests/test_spark_csv_to_parquet.sh @@ -38,7 +38,7 @@ for SPARK_VERSION in $SPARK_VERSIONS; do if ! [ -f "$tar" ]; then echo "fetching $tar" # some systems don't have wget - if which wget &>/dev/null; then + if type -P wget &>/dev/null; then wget "http://d3kbcqa49mib13.cloudfront.net/$tar" else curl -L "http://d3kbcqa49mib13.cloudfront.net/$tar" > "$tar" diff --git a/tests/test_spark_json_to_avro.sh b/tests/test_spark_json_to_avro.sh index 24bd443d5..7c511eb3a 100755 --- a/tests/test_spark_json_to_avro.sh +++ b/tests/test_spark_json_to_avro.sh @@ -41,7 +41,7 @@ for SPARK_VERSION in $SPARK_VERSIONS; do if ! [ -f "$tar" ]; then echo "fetching $tar" # some systems don't have wget - if which wget &>/dev/null; then + if type -P wget &>/dev/null; then wget "http://d3kbcqa49mib13.cloudfront.net/$tar" else curl -L "http://d3kbcqa49mib13.cloudfront.net/$tar" > "$tar" diff --git a/tests/test_spark_json_to_parquet.sh b/tests/test_spark_json_to_parquet.sh index 32754da9c..170631ba8 100755 --- a/tests/test_spark_json_to_parquet.sh +++ b/tests/test_spark_json_to_parquet.sh @@ -38,7 +38,7 @@ for SPARK_VERSION in $SPARK_VERSIONS; do if ! [ -f "$tar" ]; then echo "fetching $tar" # some systems don't have wget - if which wget &>/dev/null; then + if type -P wget &>/dev/null; then wget "http://d3kbcqa49mib13.cloudfront.net/$tar" else curl -L "http://d3kbcqa49mib13.cloudfront.net/$tar" > "$tar" diff --git a/tests/test_spark_z_avro_to_parquet.sh b/tests/test_spark_z_avro_to_parquet.sh index c9b077dcd..ed20eb34a 100755 --- a/tests/test_spark_z_avro_to_parquet.sh +++ b/tests/test_spark_z_avro_to_parquet.sh @@ -40,7 +40,7 @@ for SPARK_VERSION in $SPARK_VERSIONS; do if ! [ -f "$tar" ]; then echo "fetching $tar" # some systems don't have wget - if which wget &>/dev/null; then + if type -P wget &>/dev/null; then wget "http://d3kbcqa49mib13.cloudfront.net/$tar" else curl -L "http://d3kbcqa49mib13.cloudfront.net/$tar" > "$tar" diff --git a/tests/test_spark_z_parquet_to_avro.sh b/tests/test_spark_z_parquet_to_avro.sh index c42959cdf..be4f764f8 100755 --- a/tests/test_spark_z_parquet_to_avro.sh +++ b/tests/test_spark_z_parquet_to_avro.sh @@ -40,7 +40,7 @@ for SPARK_VERSION in $SPARK_VERSIONS; do if ! [ -f "$tar" ]; then echo "fetching $tar" # some systems don't have wget - if which wget &>/dev/null; then + if type -P wget &>/dev/null; then wget "http://d3kbcqa49mib13.cloudfront.net/$tar" else curl -L "http://d3kbcqa49mib13.cloudfront.net/$tar" > "$tar" diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index f12652267..1efa5b4e6 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -25,14 +25,14 @@ section "Testing validate_multimedia.py" export TIMEOUT=3 -if ! which ffmpeg &>/dev/null; then +if ! type -P ffmpeg &>/dev/null; then # repos are broken on both Ubuntu until 15 and RHEL/CentOS :-( # not gonna cause major compilation for this when it works on my dev systems echo "WARNING: ffmpeg not installed, skipping validate_multimedia.py tests" exit 0 - if which apt-get &>/dev/null; then + if type -P apt-get &>/dev/null; then sudo apt-get install -y ffmpeg - elif which yum &>/dev/null; then + elif type -P yum &>/dev/null; then echo "WARNING: cannot auto-install ffmpeg on RHEL/CentOS, the 3rd party repos and deps are seriously broken" fi fi From da435bc42e58f650a14bfaaf691bbcbdcc3461d7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 4 Oct 2019 11:25:07 +0100 Subject: [PATCH 0376/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7d185696f..93e41927f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7d185696f55d312ae7e4c07664fcc3dc1791738e +Subproject commit 93e41927fcd7b2965353fb439b83cdb3df75f56f From d9f0007bf985571e893807df04ae2d53ff54d8e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 4 Oct 2019 11:25:07 +0100 Subject: [PATCH 0377/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d4043ff34..49d379dbc 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d4043ff346c84b0ffe39d25d1fc6f4b680fb4f98 +Subproject commit 49d379dbc9f7e7e74a0e3e2d7a3004f9e40ff0e6 From e6da049d62027e2a8ca31067a291fdc054f41067 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 13:43:08 +0100 Subject: [PATCH 0378/2779] added validate_toml.py --- validate_toml.py | 115 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100755 validate_toml.py diff --git a/validate_toml.py b/validate_toml.py new file mode 100755 index 000000000..847545188 --- /dev/null +++ b/validate_toml.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-10-07 10:23:31 +0100 (Mon, 07 Oct 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Simple TOML file validator + +Validates each file passed as an argument + +Directories are recursed, checking all files ending in a .toml suffix. + +Works like a standard unix filter program - if no files are passed as arguments or '-' is given then reads +from standard input + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import re +import sys +import toml +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import die, log, log_option, validate_regex + from validate_ini import IniValidatorTool +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class TomlValidatorTool(IniValidatorTool): + + def __init__(self): + # Python 2.x + super(TomlValidatorTool, self).__init__() + # Python 3.x + # super().__init__() + self.re_suffix = re.compile(r'.*\.toml$', re.I) + + def add_options(self): + self.add_opt('-i', '--include', metavar='regex', default=os.getenv('INCLUDE'), + help='Regex of file / directory paths to check only ones that match ($INCLUDE, case insensitive)') + self.add_opt('-e', '--exclude', metavar='regex', default=os.getenv('EXCLUDE'), + help='Regex of file / directory paths to exclude from checking, ' + \ + '($EXCLUDE, case insensitive, takes priority over --include)') + + def process_options(self): + self.include = self.get_opt('include') + self.exclude = self.get_opt('exclude') + if self.include: + validate_regex(self.include, 'include') + self.include = re.compile(self.include, re.I) + if self.exclude: + validate_regex(self.exclude, 'exclude') + self.exclude = re.compile(self.exclude, re.I) + for key in self.opts: + log_option(key, self.opts[key]) + + @staticmethod + def check_toml(filename): + _ = toml.load(filename) + if _: + return True + return False + + def check_file(self, filename): + self.filename = filename + if self.filename == '-': + self.filename = '' + self.valid_toml_msg = '%s => TOML OK' % self.filename + self.invalid_toml_msg = '%s => TOML INVALID' % self.filename + if self.filename == '': + log.debug('toml stdin') + self.check_toml(sys.stdin) + else: + if self.is_excluded(filename): + return + if not self.is_included(filename): + return + log.debug('checking %s', self.filename) + try: + if self.check_toml(filename): + print(self.valid_toml_msg) + else: + print(self.invalid_toml_msg) + except IOError as _: + die("ERROR: %s" % _) + + +if __name__ == '__main__': + TomlValidatorTool().main() From 2dd0f787d587290d8880a5e10d548ae3be93de9d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 13:43:23 +0100 Subject: [PATCH 0379/2779] updated validate_ini2.py --- validate_ini2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_ini2.py b/validate_ini2.py index e8954d181..aaf17f87f 100755 --- a/validate_ini2.py +++ b/validate_ini2.py @@ -65,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.12.2' +__version__ = '0.12.3' class IniValidatorTool2(IniValidatorTool): @@ -75,7 +75,7 @@ def __init__(self): super(IniValidatorTool2, self).__init__() # Python 3.x # super().__init__() - self.re_ini_suffix = re.compile(r'.*\.ini$', re.I) + self.re_suffix = re.compile(r'.*\.ini$', re.I) def add_options(self): self.add_opt('-p', '--print', action='store_true', From ff4152ee7327b8f1fe406c2f28ddccac56e51fba Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 13:44:02 +0100 Subject: [PATCH 0380/2779] updated validate_ini.py --- validate_ini.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validate_ini.py b/validate_ini.py index 9026ae06c..84411046d 100755 --- a/validate_ini.py +++ b/validate_ini.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.12.1' +__version__ = '0.12.2' class IniValidatorTool(CLI): @@ -69,7 +69,7 @@ def __init__(self): # Python 3.x # super().__init__() self.filename = None - self.re_ini_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I) + self.re_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I) # In Windows ini key cannot contain equals sign = or semicolon ; # key=val or [section] self.re_ini_section = re.compile(r'^\s*\[([\w=\:\.-]+)\]\s*$') @@ -286,7 +286,7 @@ def walk(self, path): dirs[:] = [d for d in dirs if not self.is_excluded(os.path.join(root, d))] for filename in files: file_path = os.path.join(root, filename) - if self.re_ini_suffix.match(file_path): + if self.re_suffix.match(file_path): self.check_file(file_path) def check_file(self, filename): From ca87019b976a00d91a7274a6843181d07e7ee8ca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 13:57:37 +0100 Subject: [PATCH 0381/2779] updated validate_toml.py --- validate_toml.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/validate_toml.py b/validate_toml.py index 847545188..b75c2dce0 100755 --- a/validate_toml.py +++ b/validate_toml.py @@ -60,6 +60,8 @@ def __init__(self): # Python 3.x # super().__init__() self.re_suffix = re.compile(r'.*\.toml$', re.I) + self.valid_toml_msg = ' => TOML OK' + self.invalid_toml_msg = ' => TOML INVALID' def add_options(self): self.add_opt('-i', '--include', metavar='regex', default=os.getenv('INCLUDE'), @@ -82,9 +84,12 @@ def process_options(self): @staticmethod def check_toml(filename): - _ = toml.load(filename) - if _: - return True + try: + _ = toml.load(filename) + if _: + return True + except toml.decoder.TomlDecodeError: + return False return False def check_file(self, filename): @@ -107,6 +112,7 @@ def check_file(self, filename): print(self.valid_toml_msg) else: print(self.invalid_toml_msg) + sys.exit(2) except IOError as _: die("ERROR: %s" % _) From d9f12bfaf8939c7b7cebdad625562c94c9071965 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 13:58:18 +0100 Subject: [PATCH 0382/2779] added tests/test_validate_toml.sh --- tests/test_validate_toml.sh | 111 ++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100755 tests/test_validate_toml.sh diff --git a/tests/test_validate_toml.sh b/tests/test_validate_toml.sh new file mode 100755 index 000000000..3dd37ed60 --- /dev/null +++ b/tests/test_validate_toml.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd "$srcdir/.."; + +# shellcheck disable=SC1091 +. ./tests/utils.sh + +section "Testing validate_toml.py" + +export TIMEOUT=3 + +exclude='/tests/spark-\d+\.\d+.\d+-bin-hadoop\d+.\d+$|broken|error' + +data_dir="tests/data" +broken_dir="tests/toml_broken" + +if [ $# -gt 0 ]; then + echo "validate_toml.py $*" + ./validate_toml.py "$@" + echo +fi + +data_dir="tests/data" +if is_inside_docker; then + export TIMEOUT=120 +fi + +rm -fr "$broken_dir" || : +mkdir "$broken_dir" + +./validate_toml.py --exclude "$exclude" . +echo + +echo +echo "checking directory recursion (mixed with explicit file given)" +./validate_toml.py "$data_dir/test.toml" +echo + +echo "checking symlink handling" +ln -sfv "test.toml" "$data_dir/testlink.toml" +./validate_toml.py "$data_dir/testlink.toml" +rm "$data_dir/testlink.toml" +echo + +echo "checking toml file without an extension" +cp -iv "$(find "${1:-.}" -type f -iname '*.toml' | grep -v -e '/spark-.*-bin-hadoop.*/' -e 'broken' -e 'error' | head -n1)" "$broken_dir/no_extension_testfile" +./validate_toml.py -t 1 "$broken_dir/no_extension_testfile" +echo + +echo "testing stdin" +./validate_toml.py - < "$data_dir/test.toml" +./validate_toml.py < "$data_dir/test.toml" +echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 +./validate_toml.py "$data_dir/test.toml" - < "$data_dir/test.toml" +echo + +check_broken(){ + local filename="$1" + local expected_exitcode="${2:-2}" + local options="${*:3}" + set +e + # shellcheck disable=SC2086 + ./validate_toml.py -t 1 $options "$filename" + exitcode=$? + set -e + if [ "$exitcode" = "$expected_exitcode" ]; then + echo "successfully detected broken toml in '$filename', returned exit code $exitcode" + echo + #elif [ $exitcode != 0 ]; then + # echo "returned unexpected non-zero exit code $exitcode for broken toml in '$filename'" + # exit 1 + else + echo "FAILED, returned unexpected exit code $exitcode for broken toml in '$filename'" + exit 1 + fi +} + +check_broken_sample_files toml + +rm -fr "$broken_dir" + +echo + +echo "checking for non-existent file" +check_broken nonexistentfile 2 +echo + +echo "=======" +echo "SUCCESS" +echo "=======" + +echo +echo From 6ef29133630d31420816c8bbc105cb97a128253e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 13:58:23 +0100 Subject: [PATCH 0383/2779] added tests/data/test.toml --- tests/data/test.toml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/data/test.toml diff --git a/tests/data/test.toml b/tests/data/test.toml new file mode 100644 index 000000000..64d942504 --- /dev/null +++ b/tests/data/test.toml @@ -0,0 +1,25 @@ + +title = "TOML Example" + +[owner] +name = "Hari Sekhon" + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002,] +connection_max = 5000 +enabled = true + +[clients] +data = [ [ "gamma", "delta",], [ 1, 2,],] +hosts = [ "alpha", "omega",] + +[servers.alpha] +ip = "10.0.0.1" +dc = "eqdc10" + +[servers.beta] + +ip = "10.0.0.2" +dc = "eqdc10" + From e217bb0d28a646c262496285823df4521f68906e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 13:59:42 +0100 Subject: [PATCH 0384/2779] updated tests/test_validate_yaml.sh --- tests/test_validate_yaml.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_validate_yaml.sh b/tests/test_validate_yaml.sh index b0391525b..7ab5b2975 100755 --- a/tests/test_validate_yaml.sh +++ b/tests/test_validate_yaml.sh @@ -119,6 +119,8 @@ echo "checking for non-existent file" check_broken nonexistentfile 2 echo +rm -fv "$data_dir/test.yml" + echo "=======" echo "SUCCESS" echo "=======" From 95cc26ef811dc7ae747156b9bbd22118b86f2de1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 14:42:41 +0100 Subject: [PATCH 0385/2779] added tests/test_validate_cson.sh --- tests/test_validate_cson.sh | 111 ++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100755 tests/test_validate_cson.sh diff --git a/tests/test_validate_cson.sh b/tests/test_validate_cson.sh new file mode 100755 index 000000000..20b9d332b --- /dev/null +++ b/tests/test_validate_cson.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd "$srcdir/.."; + +# shellcheck disable=SC1091 +. ./tests/utils.sh + +section "Testing validate_cson.py" + +export TIMEOUT=3 + +exclude='/tests/spark-\d+\.\d+.\d+-bin-hadoop\d+.\d+$|broken|error' + +data_dir="tests/data" +broken_dir="tests/cson_broken" + +if [ $# -gt 0 ]; then + echo "validate_cson.py $*" + ./validate_cson.py "$@" + echo +fi + +data_dir="tests/data" +if is_inside_docker; then + export TIMEOUT=120 +fi + +rm -fr "$broken_dir" || : +mkdir "$broken_dir" + +./validate_cson.py --exclude "$exclude" . +echo + +echo +echo "checking directory recursion (mixed with explicit file given)" +./validate_cson.py "$data_dir/test.json" +echo + +echo "checking symlink handling" +ln -sfv "test.json" "$data_dir/testlink.json" +./validate_cson.py "$data_dir/testlink.json" +rm "$data_dir/testlink.json" +echo + +echo "checking cson file without an extension" +cp -iv "$(find "${1:-.}" -type f -iname '*.json' | grep -v -e '/spark-.*-bin-hadoop.*/' -e 'broken' -e 'error' | head -n1)" "$broken_dir/no_extension_testfile" +./validate_cson.py -t 1 "$broken_dir/no_extension_testfile" +echo + +echo "testing stdin" +./validate_cson.py - < "$data_dir/test.json" +./validate_cson.py < "$data_dir/test.json" +echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 +./validate_cson.py "$data_dir/test.json" - < "$data_dir/test.json" +echo + +check_broken(){ + local filename="$1" + local expected_exitcode="${2:-2}" + local options="${*:3}" + set +e + # shellcheck disable=SC2086 + ./validate_cson.py -t 1 $options "$filename" + exitcode=$? + set -e + if [ "$exitcode" = "$expected_exitcode" ]; then + echo "successfully detected broken cson in '$filename', returned exit code $exitcode" + echo + #elif [ $exitcode != 0 ]; then + # echo "returned unexpected non-zero exit code $exitcode for broken cson in '$filename'" + # exit 1 + else + echo "FAILED, returned unexpected exit code $exitcode for broken cson in '$filename'" + exit 1 + fi +} + +check_broken_sample_files json cson + +rm -fr "$broken_dir" + +echo + +echo "checking for non-existent file" +check_broken nonexistentfile 2 +echo + +echo "=======" +echo "SUCCESS" +echo "=======" + +echo +echo From 460209555a0750eddc68106842cf2cd98c86605a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 14:44:15 +0100 Subject: [PATCH 0386/2779] added validate_cson.py --- validate_cson.py | 128 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100755 validate_cson.py diff --git a/validate_cson.py b/validate_cson.py new file mode 100755 index 000000000..4287f6d93 --- /dev/null +++ b/validate_cson.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-10-07 10:23:31 +0100 (Mon, 07 Oct 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Simple CSON file validator + +Validates each file passed as an argument + +Directories are recursed, checking all files ending in a .cson suffix. + +Works like a standard unix filter program - if no files are passed as arguments or '-' is given then reads +from standard input + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import re +import sys +import tempfile +import cson +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import die, log, log_option, validate_regex + from validate_ini import IniValidatorTool +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class CsonValidatorTool(IniValidatorTool): + + def __init__(self): + # Python 2.x + super(CsonValidatorTool, self).__init__() + # Python 3.x + # super().__init__() + self.re_suffix = re.compile(r'.*\.cson$', re.I) + self.valid_cson_msg = ' => CSON OK' + self.invalid_cson_msg = ' => CSON INVALID' + + def add_options(self): + self.add_opt('-i', '--include', metavar='regex', default=os.getenv('INCLUDE'), + help='Regex of file / directory paths to check only ones that match ($INCLUDE, case insensitive)') + self.add_opt('-e', '--exclude', metavar='regex', default=os.getenv('EXCLUDE'), + help='Regex of file / directory paths to exclude from checking, ' + \ + '($EXCLUDE, case insensitive, takes priority over --include)') + + def process_options(self): + self.include = self.get_opt('include') + self.exclude = self.get_opt('exclude') + if self.include: + validate_regex(self.include, 'include') + self.include = re.compile(self.include, re.I) + if self.exclude: + validate_regex(self.exclude, 'exclude') + self.exclude = re.compile(self.exclude, re.I) + for key in self.opts: + log_option(key, self.opts[key]) + + @staticmethod + def check_cson(filehandle): + try: + _ = cson.load(filehandle) + if _: + return True + except ValueError: + return False + return False + + def check_file(self, filename): + self.filename = filename + if self.filename == '-': + self.filename = '' + self.valid_cson_msg = '%s => CSON OK' % self.filename + self.invalid_cson_msg = '%s => CSON INVALID' % self.filename + if self.filename == '': + log.debug('cson stdin') + tmp = tempfile.NamedTemporaryFile() + log.debug('created tmp file from stdin: %s', tmp.name) + tmp.write(sys.stdin.read()) + tmp.seek(0) + self.check_cson(tmp.name) + tmp.close() + else: + if self.is_excluded(filename): + return + if not self.is_included(filename): + return + log.debug('checking %s', self.filename) + try: + #with open(filename) as filehandle: + if self.check_cson(filename): + print(self.valid_cson_msg) + else: + print(self.invalid_cson_msg) + sys.exit(2) + except IOError as _: + die("ERROR: %s" % _) + + +if __name__ == '__main__': + CsonValidatorTool().main() From 12183a7d03e605596ffbc5f0fce0e328c62a308c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 15:43:37 +0100 Subject: [PATCH 0387/2779] updated requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 59f598b91..30adc61ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ ldif3>=3.2.2 #MarkupSafe>=0.23 #Markdown>=2.6.8 numpy>=1.8.0rc1 +python-cson>=1.0.9 psutil>=4.3.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 From 40408a8f23444df3719c28107d63681975f86f35 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Oct 2019 22:39:32 +0100 Subject: [PATCH 0388/2779] updated requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 30adc61ce..ece77890d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,5 +43,6 @@ snakebite>=2.11.0 thrift-sasl>=0.2.1 thrift>=0.9.3 thriftpy>=0.3.9 +toml>=0.10.0 xmltodict>=0.10.2 yamllint>=1.15.0 From 8d85e26e55d843cacdf6163d4534f0b3be1311bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2019 11:02:28 +0100 Subject: [PATCH 0389/2779] updated Makefile --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1488e4e57..17e531f15 100755 --- a/Makefile +++ b/Makefile @@ -29,14 +29,14 @@ # =================== -REPO := HariSekhon/DevOps-Python-tools - -CODE_FILES := $(shell find . -type f -name '*.py' -o -type f -name '*.sh' | grep -v -e bash-tools -e /pylib/) - ifneq ("$(wildcard bash-tools/Makefile.in)", "") include bash-tools/Makefile.in endif +REPO := HariSekhon/DevOps-Python-tools + +CODE_FILES := $(shell find . -type f -name '*.py' -o -type f -name '*.sh' | grep -v -e bash-tools -e /pylib/) + .PHONY: build build: @echo ========================= From 4a230366500463e920416e52ff327312f7acf484 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2019 11:02:38 +0100 Subject: [PATCH 0390/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 93e41927f..d40da9155 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 93e41927fcd7b2965353fb439b83cdb3df75f56f +Subproject commit d40da9155a82196ad30fd3db2a25a7d28c50d812 From 87b8f1aaefa9f04ad5c616a9a005bf3e1605329a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2019 11:02:38 +0100 Subject: [PATCH 0391/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 49d379dbc..ca2b67123 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 49d379dbc9f7e7e74a0e3e2d7a3004f9e40ff0e6 +Subproject commit ca2b67123f5feafb4dbe040c8970f79d826a9202 From 36b9e5c832991ea7374ae84bb4cab11fdb4be321 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2019 17:53:43 +0100 Subject: [PATCH 0392/2779] updated travis_debug_session.py --- travis_debug_session.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/travis_debug_session.py b/travis_debug_session.py index 51847ed46..6812eeead 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -30,6 +30,10 @@ URL so you can simply paste the path to a failing build and it'll just work. The switch versions of --job-id and --repo take priority as they're more explicit +Travis CI doc on debug builds: + + https://docs.travis-ci.com/user/running-build-in-debug-mode/ + """ from __future__ import absolute_import From 2d0d310584ebf86f22bd5ae9abd09eb2e1065e36 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2019 21:03:20 +0100 Subject: [PATCH 0393/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d40da9155..7c8ecb7cf 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d40da9155a82196ad30fd3db2a25a7d28c50d812 +Subproject commit 7c8ecb7cff5e733eb7cf0835912c7a416b0d30a1 From 1e4bf51028361acd0bff59f32c8546e07709a6c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2019 21:03:20 +0100 Subject: [PATCH 0394/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ca2b67123..c450e67a3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ca2b67123f5feafb4dbe040c8970f79d826a9202 +Subproject commit c450e67a319393a744ba5d639084b979470936e0 From 9e0ea38bc93e3547e13fe16b5e670598eb48841a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Oct 2019 22:39:34 +0100 Subject: [PATCH 0395/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7c8ecb7cf..03c43efff 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7c8ecb7cff5e733eb7cf0835912c7a416b0d30a1 +Subproject commit 03c43efff91c5443cbfed5006d522249e5684b1b From 599fd1a6b9621d38c0083af07ff223e43c513570 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Oct 2019 22:39:34 +0100 Subject: [PATCH 0396/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c450e67a3..c17c1f109 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c450e67a319393a744ba5d639084b979470936e0 +Subproject commit c17c1f109c498bf92a08d574c282e0b642f2757f From e5d1a838024612ffd2caf58f3f11f735388e24ad Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Oct 2019 23:06:29 +0100 Subject: [PATCH 0397/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c17c1f109..4d5449878 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c17c1f109c498bf92a08d574c282e0b642f2757f +Subproject commit 4d5449878a6dacdb491f7d1fc30a2aabbb2978a5 From 22c111bfbad0245cfa3dd43319469761368a7b9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Oct 2019 23:10:27 +0100 Subject: [PATCH 0398/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 03c43efff..c600ebfe8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 03c43efff91c5443cbfed5006d522249e5684b1b +Subproject commit c600ebfe8baeb403fd3fec55aabd3a573d1cec7a From 062612bf8fe755657fdc222827414bf17914d593 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Oct 2019 23:10:27 +0100 Subject: [PATCH 0399/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4d5449878..4afb8da42 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4d5449878a6dacdb491f7d1fc30a2aabbb2978a5 +Subproject commit 4afb8da42d1430d44709613416dfe9d5577bfec2 From 4197d9a92c64e8eeddb28bc8c59860bcb7ad0105 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Oct 2019 00:09:07 +0100 Subject: [PATCH 0400/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c600ebfe8..98956885a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c600ebfe8baeb403fd3fec55aabd3a573d1cec7a +Subproject commit 98956885a36bf1a6c2a842dbb823913b9c8a1f66 From b6a11c2eecf651f9e3d69d9674a12b8f08e93bf8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Oct 2019 00:09:07 +0100 Subject: [PATCH 0401/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4afb8da42..02b0a343b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4afb8da42d1430d44709613416dfe9d5577bfec2 +Subproject commit 02b0a343b055dfd6bed4d64c40aa77ed8394c1b2 From d48ab8b1631697021cce6569e5b9ab6218cf5024 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 13 Oct 2019 18:41:46 +0100 Subject: [PATCH 0402/2779] fixed old method rename call so that we can call on globs of filenames too --- find_duplicate_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 9b2c50d1e..00b45fe2e 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -81,7 +81,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.3' +__version__ = '0.5.4' class FindDuplicateFiles(CLI): @@ -270,7 +270,7 @@ def check_path(self, path): # os.walk returns nothing if path is a file, and must store file names, sizes, checksums and regex captures # even for standalone file args if os.path.isfile(path): - self.check_file(path) + self.is_file_dup(path) elif os.path.isdir(path): # returns generator # root is the dir, dirs and files are child basenames From 646e854389c9e01ae3fc522f195c609e8255308e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Oct 2019 15:19:54 +0100 Subject: [PATCH 0403/2779] switched to pinned versions --- requirements.txt | 70 ++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/requirements.txt b/requirements.txt index ece77890d..21c54e8a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,48 +1,48 @@ -avro>=1.8.1 -# requires Python >= 3.4, build in Makefile instead -#avro-python3>=1.9.0 -awscli>=1.16.241 -#bitarray>=0.8.1 -#cassandra-driver>=3.6.0 -dicttoxml>=1.7.4 +avro==1.8.1 +# requires Python == 3.4, build in Makefile instead +#avro-python3==1.9.0 +awscli==1.16.241 +#bitarray==0.8.1 +#cassandra-driver==3.6.0 +dicttoxml==1.7.4 # Elasticsearch library must match major version :( # for Elasticsearch 2.x #elasticsearch>=2.0.0,<3.0.0 # for Elasticsearch 1.x #elasticsearch>=1.0.0,<2.0.0 # fails on requiring newer version of setuptools -#Flask>=0.10.1 -GitPython>=2.0.8 -happybase>=1.0.0 -humanize>=0.5.1 -#impyla>=0.12.0 -Jinja2>=2.8 -#kazoo>=2.2.1 -ldif3>=3.2.2 -#MarkupSafe>=0.23 -#Markdown>=2.6.8 -numpy>=1.8.0rc1 -python-cson>=1.0.9 -psutil>=4.3.0 +#Flask==0.10.1 +GitPython==2.0.8 +happybase==1.0.0 +humanize==0.5.1 +#impyla==0.12.0 +Jinja2==2.8 +#kazoo==2.2.1 +ldif3==3.2.2 +#MarkupSafe==0.23 +#Markdown==2.6.8 +numpy==1.8.0rc1 +python-cson==1.0.9 +psutil==4.3.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 -python-ldap>=3.2.0 +python-ldap==3.2.0 # PyLint fails some checks in Travis that pass locally -#pyhs2>=0.6.0 -#python-jenkins>=0.4.13 +#pyhs2==0.6.0 +#python-jenkins==0.4.13 # pulled in automatically by snakebite[kerberos] -#python-krbV>=1.0.90 +#python-krbV==1.0.90 # needed by avro -python-snappy>=0.5 -sasl>=0.2.1 -sh>=1.12.14 +python-snappy==0.5 +sasl==0.2.1 +sh==1.12.14 # pulls in python-KrbV as a dependency which doesn't build on Mac any more # moved to Makefile as best effort -#snakebite[kerberos]>=2.11.0 -snakebite>=2.11.0 -thrift-sasl>=0.2.1 -thrift>=0.9.3 -thriftpy>=0.3.9 -toml>=0.10.0 -xmltodict>=0.10.2 -yamllint>=1.15.0 +#snakebite[kerberos]==2.11.0 +snakebite==2.11.0 +thrift-sasl==0.2.1 +thrift==0.9.3 +thriftpy==0.3.9 +toml==0.10.0 +xmltodict==0.10.2 +yamllint==1.15.0 From eef30fd096a33d64f129d3feb4d7d0a4b0ebc46c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Oct 2019 15:20:01 +0100 Subject: [PATCH 0404/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 98956885a..eb301fe34 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 98956885a36bf1a6c2a842dbb823913b9c8a1f66 +Subproject commit eb301fe34149b5ddddcfe7f49f4d0a284f70b2cf From 393d658412e23df5fa81338c4f1a7af56d332303 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Oct 2019 15:20:02 +0100 Subject: [PATCH 0405/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 02b0a343b..84170fd44 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 02b0a343b055dfd6bed4d64c40aa77ed8394c1b2 +Subproject commit 84170fd442f5de6f3f855f5a737b4676a27345b3 From c38f2f4516058327d31e0de7fe365ca417fd7e2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Oct 2019 16:34:06 +0100 Subject: [PATCH 0406/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 21c54e8a0..ba729233b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ Jinja2==2.8 ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 -numpy==1.8.0rc1 +numpy==1.8.0 python-cson==1.0.9 psutil==4.3.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools From a0d03850fcdfdfbfb95e91b1a234dae567985857 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Oct 2019 16:35:23 +0100 Subject: [PATCH 0407/2779] updated Makefile --- Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile b/Makefile index 17e531f15..8b1ef9455 100755 --- a/Makefile +++ b/Makefile @@ -157,10 +157,6 @@ jython: if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi sh jython_install.sh -.PHONY: pycompile -pycompile: - bash-tools/python_compile.sh - .PHONY: test-lib test-lib: cd pylib && $(MAKE) test From 54ca6bb82a479e4822b531627921548a8c8889e2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Oct 2019 23:00:08 +0100 Subject: [PATCH 0408/2779] added .pyup.yml --- .pyup.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .pyup.yml diff --git a/.pyup.yml b/.pyup.yml new file mode 100644 index 000000000..aabbdd2ef --- /dev/null +++ b/.pyup.yml @@ -0,0 +1,4 @@ +# https://pyup.io/docs/configuration/ + +update: insecure +assignees: HariSekhon From 0ac61cff09c091420d947f3b56cbd96c26351726 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 16 Oct 2019 23:06:47 +0100 Subject: [PATCH 0409/2779] Update numpy from 1.8.0 to 1.17.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ba729233b..f77370e2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ Jinja2==2.8 ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 -numpy==1.8.0 +numpy==1.17.2 python-cson==1.0.9 psutil==4.3.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools From 3102e5f3c03f30931ff40c1a2fd0fdbb8b737b44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Oct 2019 23:08:54 +0100 Subject: [PATCH 0410/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 07dec23e6..aed63b700 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ Hari Sekhon - DevOps Python Tools [![Codacy Badge](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) +[![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) From a69b4579117ba55ca8f8ce004f008487bad5745c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Oct 2019 11:18:34 +0100 Subject: [PATCH 0411/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index eb301fe34..7ae001060 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit eb301fe34149b5ddddcfe7f49f4d0a284f70b2cf +Subproject commit 7ae001060e4d6a0c628325f51d6f567cf372a751 From 992adbef251e3536e0c645a760be1440132dd269 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Oct 2019 11:18:34 +0100 Subject: [PATCH 0412/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 84170fd44..5a739b654 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 84170fd442f5de6f3f855f5a737b4676a27345b3 +Subproject commit 5a739b6540ea334c96a2a58644dce0d1306c5b4b From 00973b83d64675709ac7cb5ce9285dca557c787d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Oct 2019 11:22:52 +0100 Subject: [PATCH 0413/2779] updated requirements.txt --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f77370e2f..1507d24ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,9 @@ Jinja2==2.8 ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 -numpy==1.17.2 +# Python 3.5+ +#numpy==1.17.2 +numpy==1.16.5 python-cson==1.0.9 psutil==4.3.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools From 2701a7be66059be443e1bde5bb134a4c892e3e14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Oct 2019 11:23:39 +0100 Subject: [PATCH 0414/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1507d24ba..6967c1871 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ GitPython==2.0.8 happybase==1.0.0 humanize==0.5.1 #impyla==0.12.0 -Jinja2==2.8 +Jinja2==2.10.1 #kazoo==2.2.1 ldif3==3.2.2 #MarkupSafe==0.23 From 7c24286d8ead6fc2c5547395c31a20a7384d41d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Oct 2019 11:49:13 +0100 Subject: [PATCH 0415/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index aed63b700..2e94b11df 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Hari Sekhon - DevOps Python Tools [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) +[![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) [![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) From 05d8a48f1b13086a16329b41ca14b5662aaf7f4f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Oct 2019 11:51:30 +0100 Subject: [PATCH 0416/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2e94b11df..7e848a889 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ Hari Sekhon - DevOps Python Tools [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) -[![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) -[![](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) +[![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) +[![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) ### Hadoop, Spark / PySpark, HBase, Pig, Ambari, IPython and Linux Tools ### From 9631b31831e30941a32e81a1344aa76a36ba6cff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Oct 2019 14:22:11 +0100 Subject: [PATCH 0417/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7ae001060..05ba9cae4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7ae001060e4d6a0c628325f51d6f567cf372a751 +Subproject commit 05ba9cae4200ae40997239ca4532b4788fae98ea From d8d797712c17dea5e2948489d851a1fd2578947b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Oct 2019 14:22:11 +0100 Subject: [PATCH 0418/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5a739b654..ce109e54e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5a739b6540ea334c96a2a58644dce0d1306c5b4b +Subproject commit ce109e54e72f4bedb12e00e8a722a0a843703d7f From 5ee19586af60a29378d4b2c6c0feb16178c4a96b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 11:27:55 +0100 Subject: [PATCH 0419/2779] updated Makefile --- Makefile | 77 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 8b1ef9455..ce71ddcd3 100755 --- a/Makefile +++ b/Makefile @@ -37,6 +37,14 @@ REPO := HariSekhon/DevOps-Python-tools CODE_FILES := $(shell find . -type f -name '*.py' -o -type f -name '*.sh' | grep -v -e bash-tools -e /pylib/) +# placeholders to silence check_makefile.sh warnings - should be set in client Makefiles after sourcing +ifndef PARQUET_VERSION + PARQUET_VERSION := +endif +ifndef SKIP_PARQUET + SKIP_PARQUET := +endif + .PHONY: build build: @echo ========================= @@ -57,9 +65,9 @@ init: .PHONY: python python: cd pylib && $(MAKE) - # don't pull parquet tools in to docker image by default, will bloat it - # can fetch separately by running 'make parquet-tools' if you really want to - if [ -f /.dockerenv -o -n "$(SKIP_PARQUET)" ]; then \ + @# don't pull parquet tools in to docker image by default, will bloat it + @# can fetch separately by running 'make parquet-tools' if you really want to + @if [ -f /.dockerenv -o -n "$(SKIP_PARQUET)" ]; then \ echo; echo; \ echo "Skipping Parquet install..."; \ echo; echo; \ @@ -67,45 +75,44 @@ python: $(MAKE) parquet-tools; \ fi - # only install pip packages not installed via system packages - #$(SUDO_PIP) pip install --upgrade -r requirements.txt - #$(SUDO_PIP) pip install -r requirements.txt + @# only install pip packages not installed via system packages + @#$(SUDO_PIP) pip install --upgrade -r requirements.txt + @#$(SUDO_PIP) pip install -r requirements.txt @bash-tools/python_pip_install_if_absent.sh requirements.txt - # python-krbV dependency doesn't build on Mac any more and is unmaintained - # python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit - #@bash-tools/python_pip_install.sh snakebite[kerberos] || : + @# python-krbV dependency doesn't build on Mac any more and is unmaintained + @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit @bash-tools/python_pip_install.sh snakebite[kerberos] || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python @if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi - # for impyla - #$(SUDO_PIP) pip install --upgrade setuptools || : - # - # snappy may fail to install on Mac not finding snappy-c.h - workaround: - # - # brew install snappy - # - # find /usr/local -name snappy-c.h - # - # /usr/local/include/snappy-c.h - # - # sudo su - # LD_RUN_PATH=/usr/local/include pip install snappy - # - #$(SUDO_PIP) pip install --upgrade -r requirements.txt - - # for ipython-notebook-pyspark.py - #$(SUDO_PIP) pip install jinja2 - # HiveServer2 - #$(SUDO_PIP) pip install pyhs2 - # Impala - #$(SUDO_PIP) pip install impyla - # must downgrade happybase library to work on Python 2.6 - #if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi - - # Python >= 2.7 - won't build on 2.6, handle separately and accept failure + @# for impyla + @#$(SUDO_PIP) pip install --upgrade setuptools || : + @# + @# snappy may fail to install on Mac not finding snappy-c.h - workaround: + @# + @# brew install snappy + @# + @# find /usr/local -name snappy-c.h + @# + @# /usr/local/include/snappy-c.h + @# + @# sudo su + @# LD_RUN_PATH=/usr/local/include pip install snappy + @# + @#$(SUDO_PIP) pip install --upgrade -r requirements.txt + + @# for ipython-notebook-pyspark.py + @#$(SUDO_PIP) pip install jinja2 + @# HiveServer2 + @#$(SUDO_PIP) pip install pyhs2 + @# Impala + @#$(SUDO_PIP) pip install impyla + @# must downgrade happybase library to work on Python 2.6 + @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi + + @# Python >= 2.7 - won't build on 2.6, handle separately and accept failure @bash-tools/python_pip_install.sh "ipython[notebook]" || : @echo $(MAKE) pycompile From 650180a7e162346b275e345ba99e6f55e94a24f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 11:58:05 +0100 Subject: [PATCH 0420/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 05ba9cae4..ad9a494d6 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 05ba9cae4200ae40997239ca4532b4788fae98ea +Subproject commit ad9a494d6599179f554954af6b9448fbdb41eb21 From 94c42b06e5a0804c140f0ad782197e1c53c200a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 11:58:05 +0100 Subject: [PATCH 0421/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ce109e54e..0ef1f5385 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ce109e54e72f4bedb12e00e8a722a0a843703d7f +Subproject commit 0ef1f5385332e6e71d0758e4cfcca0fbb57e29c3 From a4b4b0d83c0083da90e4e415ae669c3bcfcd560f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 13:25:34 +0100 Subject: [PATCH 0422/2779] added potential fix for #8 --- ambari_trigger_service_checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ambari_trigger_service_checks.py b/ambari_trigger_service_checks.py index abe593da1..edf98b2bf 100755 --- a/ambari_trigger_service_checks.py +++ b/ambari_trigger_service_checks.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.1' +__version__ = '0.2.2' class AmbariTriggerServiceChecks(CLI): @@ -307,7 +307,7 @@ def watch_scheduled_request(self, request_schedule_id): def parse_scheduled_request(content): try: _ = json.loads(content) - if _['RequestSchedule']['last_execution_status'] == 'COMPLETED': + if _['RequestSchedule']['status'] == 'COMPLETED': log.info('COMPLETED') return 'COMPLETED' for item in _['RequestSchedule']['batch']['batch_requests']: From b2e3b13fd48f8550b4ef3c73c7790bf88010674b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 13:34:42 +0100 Subject: [PATCH 0423/2779] updated .travis.yml --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index ccbeb9cea..154aa8978 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,6 +57,11 @@ cache: pip services: - docker +# avoid package checksum mismatches when installing packages +before_install: + - sudo rm -f "${TRAVIS_ROOT}/etc/apt/apt.conf.d/99-travis-apt-proxy" + - sudo rm -rf "${TRAVIS_ROOT}/var/lib/apt/lists/"* + install: - travis_retry make #- pip install --upgrade ndg-httpsclient From 650ce3366f40aa1222d1772de50f8cdbe2aab392 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 13:34:48 +0100 Subject: [PATCH 0424/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ad9a494d6..c4ff07970 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ad9a494d6599179f554954af6b9448fbdb41eb21 +Subproject commit c4ff0797061f8fa4f50b504286e7b23966624230 From 0e9b503d0926096884e155ddeb46051947836194 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 13:34:48 +0100 Subject: [PATCH 0425/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0ef1f5385..07379ffae 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0ef1f5385332e6e71d0758e4cfcca0fbb57e29c3 +Subproject commit 07379ffae01e48ddfe4b5951503f222df0d177e9 From 088638b965a765fefeb32ad90ab17489262c198c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 14:24:28 +0100 Subject: [PATCH 0426/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c4ff07970..8dbe9ac67 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c4ff0797061f8fa4f50b504286e7b23966624230 +Subproject commit 8dbe9ac67ba555ea55e478f8ae369b18ab7406fe From 92852fe46bda7276f23bb3a3a1299f8bc13df9c6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 14:24:28 +0100 Subject: [PATCH 0427/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 07379ffae..459f6903f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 07379ffae01e48ddfe4b5951503f222df0d177e9 +Subproject commit 459f6903fa7206ad4a115cc3a18f4efeb5dedad6 From 58fb252cf10bcfea510e625238079768b385a050 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 14:51:32 +0100 Subject: [PATCH 0428/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8dbe9ac67..616fe737d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8dbe9ac67ba555ea55e478f8ae369b18ab7406fe +Subproject commit 616fe737dbd7fa0c97f899801efbb0feaa42120b From c21b691904b4d3545c939856ada60e296082fe41 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 14:51:32 +0100 Subject: [PATCH 0429/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 459f6903f..19db97899 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 459f6903fa7206ad4a115cc3a18f4efeb5dedad6 +Subproject commit 19db978992b6bd3aa7d490d2aff2372a28d59e2d From 5b9f35436fa2b06078784d8960d4d61ad3a411f2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 15:11:23 +0100 Subject: [PATCH 0430/2779] updated Makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index ce71ddcd3..66ed6ad5a 100755 --- a/Makefile +++ b/Makefile @@ -190,6 +190,7 @@ clean: cd pylib && $(MAKE) clean @# the xargs option to ignore blank input doesn't work on Mac @find . -iname '*.py[co]' -o -iname '*.jy[co]' | xargs rm -f + @find . -iname '*.spec' | xargs rm -f @find . -type d -ipath '*/tests/*' -iname 'test-*spark*.avro' | xargs rm -rf @find . -type d -ipath '*/tests/*' -iname 'test-*spark*.parquet' | xargs rm -rf @rm -f parquet-tools-$(PARQUET_VERSION)-bin.zip From d76c8659a175bb434414da552c85386389bf8197 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 15:13:41 +0100 Subject: [PATCH 0431/2779] updated hadoop_hdfs_files_native_checksums.jy --- hadoop_hdfs_files_native_checksums.jy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop_hdfs_files_native_checksums.jy b/hadoop_hdfs_files_native_checksums.jy index a1bc67622..192ecfe6d 100755 --- a/hadoop_hdfs_files_native_checksums.jy +++ b/hadoop_hdfs_files_native_checksums.jy @@ -58,7 +58,7 @@ Hari Sekhon - https://github.com/harisekhon/devops-python-tools Jython program to fetch the HDFS native checksums for given files / all files under given directories. -Quick way of checking for duplicate files but not checking content itself. This is at least ~100x more efficient in terms of data transfer than 'hadoop fs -cat | md5sum'. +Quick way of checking for duplicate files but not checking content itself. This is ~90% more efficient in terms of data transfer than 'hadoop fs -cat | md5sum'. Caveat: files with differing block sizes will not match Will be implemented in hadoop-9209 jira for versions 3.0.0, 0.23.7, 2.1.0-beta using a new command line param: hadoop fs -checksum From 4f2eb93d4c8d1d9ab8dbbe404f0fda971caecb26 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Oct 2019 15:14:10 +0100 Subject: [PATCH 0432/2779] updated hadoop_hdfs_files_native_checksums.jy --- hadoop_hdfs_files_native_checksums.jy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop_hdfs_files_native_checksums.jy b/hadoop_hdfs_files_native_checksums.jy index 192ecfe6d..6aa6e6ff2 100755 --- a/hadoop_hdfs_files_native_checksums.jy +++ b/hadoop_hdfs_files_native_checksums.jy @@ -58,7 +58,7 @@ Hari Sekhon - https://github.com/harisekhon/devops-python-tools Jython program to fetch the HDFS native checksums for given files / all files under given directories. -Quick way of checking for duplicate files but not checking content itself. This is ~90% more efficient in terms of data transfer than 'hadoop fs -cat | md5sum'. +Quick way of checking for duplicate files but not checking content itself. This is ~10x more efficient in terms of data transfer than 'hadoop fs -cat | md5sum'. Caveat: files with differing block sizes will not match Will be implemented in hadoop-9209 jira for versions 3.0.0, 0.23.7, 2.1.0-beta using a new command line param: hadoop fs -checksum From 7bc20f3b1be203221e2d7b8e99b2eb3b7eebd5a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 19 Oct 2019 22:52:50 +0100 Subject: [PATCH 0433/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 616fe737d..5b9603c5b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 616fe737dbd7fa0c97f899801efbb0feaa42120b +Subproject commit 5b9603c5ba655283f5a50cc6642b80e1126da9be From 5bb0a630499a1ed8aad08dd682dfdbb9462cf083 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 19 Oct 2019 22:52:50 +0100 Subject: [PATCH 0434/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 19db97899..b3953c0ae 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 19db978992b6bd3aa7d490d2aff2372a28d59e2d +Subproject commit b3953c0ae7f08d83eb66be1c980bcfed55e3585e From 02dffe6c8e5bfe35c45f49be303ddaec539195e2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 20 Oct 2019 19:42:48 +0100 Subject: [PATCH 0435/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index ce87a6b69..20ff7fa24 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -427,6 +427,9 @@ dest[110]="es.xpack.password=" src[111]="127.0.0.1" dest[111]="127.0.0.1" +src[112]="travis token: Abc123" +dest[112]="travis token: " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From f23c72c39873365410ceef0fcc91800c8e3a24f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 20 Oct 2019 19:42:51 +0100 Subject: [PATCH 0436/2779] updated anonymize.py --- anonymize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index e5fdc3b28..883b3c6a3 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.4' +__version__ = '0.9.5' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -353,6 +353,7 @@ def __init__(self): 'http_auth': r'(https?:\/\/)[^:]+:[^\@]*\@', 'http_auth2': r'(Proxy auth using \w+ with user )([\'"]).+([\'"])', 'http_auth3': r'\bAuthorization:\s+Basic\s+[A-Za-z0-9]+', + 'http_auth4': r'\b(token:\s+)[A-Za-z0-9]+', 'cisco': r'username .+ (?:password|secret) .*?$', 'cisco2': r'password .*?$', 'cisco3': r'\ssecret\s.*?$', @@ -425,6 +426,7 @@ def __init__(self): 'http_auth': r'$1:@', 'http_auth2': r'\1\'\2\3/', 'http_auth3': r'Authorization: Basic ', + 'http_auth4': r'\1', 'cisco': r'username password ', 'cisco2': r'password ', 'cisco3': r'secret ', From 46ef371b15c330600822df448498767450c809c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 20 Oct 2019 19:45:23 +0100 Subject: [PATCH 0437/2779] updated anonymize.py --- anonymize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 883b3c6a3..74cb19226 100755 --- a/anonymize.py +++ b/anonymize.py @@ -353,7 +353,7 @@ def __init__(self): 'http_auth': r'(https?:\/\/)[^:]+:[^\@]*\@', 'http_auth2': r'(Proxy auth using \w+ with user )([\'"]).+([\'"])', 'http_auth3': r'\bAuthorization:\s+Basic\s+[A-Za-z0-9]+', - 'http_auth4': r'\b(token:\s+)[A-Za-z0-9]+', + 'http_auth4': r'(\btoken:\s+)[A-Za-z0-9]+', 'cisco': r'username .+ (?:password|secret) .*?$', 'cisco2': r'password .*?$', 'cisco3': r'\ssecret\s.*?$', From dde1d176bad47f2051803b43eb8dec1f72181d8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 20 Oct 2019 23:32:59 +0100 Subject: [PATCH 0438/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6967c1871..3ce1b0312 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ dicttoxml==1.7.4 #elasticsearch>=1.0.0,<2.0.0 # fails on requiring newer version of setuptools #Flask==0.10.1 -GitPython==2.0.8 +GitPython==2.1.14 happybase==1.0.0 humanize==0.5.1 #impyla==0.12.0 From 4f02d17285059822e7fcdeac2ada2de7eac061f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 21 Oct 2019 15:57:26 +0100 Subject: [PATCH 0439/2779] added .flake8 --- .flake8 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..e31fb949f --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = E265,E402,F401 +max-line-length = 120 +exclude = test*/* +max-complexity = 10 From 7d0e0bec86047e4273f8424cf37024c75df94b6a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Oct 2019 21:34:00 +0100 Subject: [PATCH 0440/2779] updated README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e848a889..5b48f2202 100644 --- a/README.md +++ b/README.md @@ -319,10 +319,12 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ tools for Hadoop, NoSQL, Solr, Elasticsearch, Pig, Hive, Web URL + Nginx stats watchers, SQL and NoSQL syntax recasers, various Linux CLI tools +* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps tools for Hadoop, NoSQL, Solr, Elasticsearch, Pig, Hive, Web URL + Nginx stats watchers, SQL and NoSQL syntax recasers, various Linux CLI tools * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 400+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 80+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies + * [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule * [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library From f04c41b2cbcf55d405e4d6c1f84364f76579ec5c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Oct 2019 21:34:28 +0100 Subject: [PATCH 0441/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b48f2202..25b1d880c 100644 --- a/README.md +++ b/README.md @@ -321,7 +321,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps tools for Hadoop, NoSQL, Solr, Elasticsearch, Pig, Hive, Web URL + Nginx stats watchers, SQL and NoSQL syntax recasers, various Linux CLI tools -* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 400+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) * [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 80+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies From 327e302845943056ba4931517a564ac9446a261a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Oct 2019 22:18:39 +0100 Subject: [PATCH 0442/2779] updated README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 25b1d880c..a49b16f84 100644 --- a/README.md +++ b/README.md @@ -319,12 +319,14 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps tools for Hadoop, NoSQL, Solr, Elasticsearch, Pig, Hive, Web URL + Nginx stats watchers, SQL and NoSQL syntax recasers, various Linux CLI tools +* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (Hive, Impala, MySQL, PostgreSQL, Cassandra CQL, Apache Drill, Couchbase N1QL, Microsoft SQL Server, Oracle, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... -* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Hadoop, Docker, Kafka, Elasticsearch, RabbitMQ, Redis, HBase, Solr, Cassandra, ZooKeeper, HDFS, Yarn, Hive, Presto, Drill, Impala, Consul, Spark, Jenkins, Travis CI, Git, MySQL, Linux, DNS, Whois, SSL Certs, Yum Security Updates, Kubernetes, Mesos, Riak, MongoDB, Memcached, Couchbase, CouchDB, Neo4j, Ambari, Cloudera, Hortonworks, MapR etc. * [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 80+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies +* [HAProxy-configs](https://github.com/harisekhon/haproxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. + * [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule * [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library From 2a903942cc50fdbb5e72d03df4c883ce7f75b85b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Oct 2019 22:25:31 +0100 Subject: [PATCH 0443/2779] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a49b16f84..c54e91338 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,8 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [HAProxy-configs](https://github.com/harisekhon/haproxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. +* [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 + * [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule * [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library From c762d0acd84eb643dfbde0ac39e65dee39203519 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Oct 2019 16:06:44 +0100 Subject: [PATCH 0444/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 66ed6ad5a..7f554276d 100755 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ test: test-lib .PHONY: basic-test basic-test: test-lib - bash-tools/all.sh + bash-tools/check_all.sh .PHONY: test2 test2: From 1d16d98e96ad94bc3a4b4c146cd2c72e5249a4b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Oct 2019 16:06:52 +0100 Subject: [PATCH 0445/2779] updated all.sh --- tests/all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all.sh b/tests/all.sh index 15e3af28d..fabd97170 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -28,7 +28,7 @@ section "Running PyTools ALL" # runs against . by default cd "$srcdir/.."; -bash-tools/all.sh +bash-tools/check_all.sh tests/test_yamllint.sh From a546f4980a3449a24320b6c7fd061773176e4547 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Oct 2019 16:07:03 +0100 Subject: [PATCH 0446/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 5b9603c5b..0957efce7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 5b9603c5ba655283f5a50cc6642b80e1126da9be +Subproject commit 0957efce790de219318412ab4461ac2debc28727 From 9cb7895b0da8c8864c80e1ddf858b9dc2a525f7e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Oct 2019 16:07:04 +0100 Subject: [PATCH 0447/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b3953c0ae..54b3c04be 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b3953c0ae7f08d83eb66be1c980bcfed55e3585e +Subproject commit 54b3c04beb0501ac823cd29f22f91b89eabe8319 From cd061bcd6efe430201bc7c1d7647b888e0d8efc6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Oct 2019 17:07:07 +0100 Subject: [PATCH 0448/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0957efce7..b071e1fdc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0957efce790de219318412ab4461ac2debc28727 +Subproject commit b071e1fdc026b3204c256c279706500f3d536f78 From 1b4d78c58b47bfa94c8fe5f436a59c2e99608c73 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Oct 2019 17:07:08 +0100 Subject: [PATCH 0449/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 54b3c04be..461f71292 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 54b3c04beb0501ac823cd29f22f91b89eabe8319 +Subproject commit 461f7129227b2b207ede19c31ed00ceb50793e96 From 977e7a282e116d40b2014fe320cca5df58a3eaac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Oct 2019 17:48:25 +0100 Subject: [PATCH 0450/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b071e1fdc..6b034a1ac 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b071e1fdc026b3204c256c279706500f3d536f78 +Subproject commit 6b034a1acb99f5d1686a496fec900c306e2874ce From 8ed8688f0dff7199282cadb8a0a7eb473533b5ac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Oct 2019 17:48:26 +0100 Subject: [PATCH 0451/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 461f71292..3b9bcd0cf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 461f7129227b2b207ede19c31ed00ceb50793e96 +Subproject commit 3b9bcd0cfad6e51d9f741a1ac3494e83436f5b6e From 5467c7288361435135383f2d260cf7c577ce6e04 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Oct 2019 11:58:05 +0100 Subject: [PATCH 0452/2779] updated .travis.yml --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 154aa8978..0ef2d390c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ # https://www.linkedin.com/in/harisekhon # +version: ~> 1.0 + # Mac Python 2.7 not available - see https://travis-ci.org/HariSekhon/pylib/jobs/584226228 #os: # - linux From 08e0eac7437f2196b394e26c2285353793cdec9f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 28 Oct 2019 17:39:31 +0000 Subject: [PATCH 0453/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c54e91338..b02118d1e 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ mv -vf pylib pytools/ Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). -All PyPI modules are listed in the ```requirements.txt``` file. +All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files. Internal Mirror example ([JFrog Artifactory](https://jfrog.com/artifactory/) or similar): From dcd693233beb0944b2af75ffabe6fa3d3ffc01d7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 28 Oct 2019 17:41:31 +0000 Subject: [PATCH 0454/2779] updated README.md --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b02118d1e..3ea812319 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,13 @@ sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt ##### Mac OS X -The automated build also works on Mac OS X but you'll need to download and install [Apple XCode](https://developer.apple.com/download/). I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): +The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing `git` is enough to trigger Xcode install). + +I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): + +``` +bash-tools/setup/install_homebrew.sh +``` ``` brew install openssl wget From 2b477a782b1f9be6af986dd758fce17a20f2385c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 29 Oct 2019 11:23:36 +0000 Subject: [PATCH 0455/2779] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7f554276d..6c486a768 100755 --- a/Makefile +++ b/Makefile @@ -205,8 +205,8 @@ spark-deps: rm -vf spark-deps.zip zip spark-deps.zip pylib -.PHONY: dockerhub -dockerhub: +.PHONY: dockerhub-trigger +dockerhub-trigger: # PyTools curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/d470810b-9a44-4abc-92cc-c903a6afd962/trigger/0e69c39f-ea1b-43c7-a97d-cef1252f1400/call/ # Alpine Github From 2527b38dad29701cecf8851b584c23c6a1739255 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 31 Oct 2019 10:23:51 +0000 Subject: [PATCH 0456/2779] updated README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3ea812319..66d9791a6 100644 --- a/README.md +++ b/README.md @@ -344,3 +344,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: * https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb + +### Stargazers over time + +[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) From ec626029b18f1520a6397e99e8ddd01313026faa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Nov 2019 23:17:53 +0000 Subject: [PATCH 0457/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6b034a1ac..e7e584f70 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6b034a1acb99f5d1686a496fec900c306e2874ce +Subproject commit e7e584f701743d7b3ba0964a162e9c1b400e124f From 0895507414ec5e8202bf1cb1ab87750b20d4afff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Nov 2019 23:17:59 +0000 Subject: [PATCH 0458/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3b9bcd0cf..7f545d4b8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3b9bcd0cfad6e51d9f741a1ac3494e83436f5b6e +Subproject commit 7f545d4b8d1e33efd95c9cbf9d9d7cbeb2d539df From 18c7dd26585b41893516738ec38eeeca9b82241e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Nov 2019 23:45:58 +0000 Subject: [PATCH 0459/2779] updated bash-tools/ --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e7e584f70..9d34da1b2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e7e584f701743d7b3ba0964a162e9c1b400e124f +Subproject commit 9d34da1b25ebdf717cdc4b310dfe2224a3322d13 From 0130fe9557de366a2c400b984d1f12d8f33615dc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Nov 2019 23:46:14 +0000 Subject: [PATCH 0460/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7f545d4b8..ddbca888b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7f545d4b8d1e33efd95c9cbf9d9d7cbeb2d539df +Subproject commit ddbca888bc224295015f4e386cb245a26557c970 From 02492335c0fa700dd8f69d23ceeeae10dc32c58f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 2 Nov 2019 11:21:32 +0000 Subject: [PATCH 0461/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9d34da1b2..cea4c6fae 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9d34da1b25ebdf717cdc4b310dfe2224a3322d13 +Subproject commit cea4c6fae729d43015eb861fbf52f03abe52f2c0 From c2e5d8a0802fc80964c34ed115123f18e94c0dca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 2 Nov 2019 11:21:32 +0000 Subject: [PATCH 0462/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ddbca888b..93cc037e0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ddbca888bc224295015f4e386cb245a26557c970 +Subproject commit 93cc037e06a8f4c360369e0e76a0550dc2822976 From 28bd067e4209344c42d7cbe4a4852d5287337f52 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 15:15:02 +0000 Subject: [PATCH 0463/2779] updated requirements.txt --- requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3ce1b0312..203addee6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ dicttoxml==1.7.4 GitPython==2.1.14 happybase==1.0.0 humanize==0.5.1 -#impyla==0.12.0 +#impyla==0.16.0 Jinja2==2.10.1 #kazoo==2.2.1 ldif3==3.2.2 @@ -29,8 +29,6 @@ psutil==4.3.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 python-ldap==3.2.0 -# PyLint fails some checks in Travis that pass locally -#pyhs2==0.6.0 #python-jenkins==0.4.13 # pulled in automatically by snakebite[kerberos] #python-krbV==1.0.90 From 771b36bbd5ee2c473f6099132f9807e1d4f62e51 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 15:22:11 +0000 Subject: [PATCH 0464/2779] updated requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 203addee6..270289919 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,8 @@ python-cson==1.0.9 psutil==4.3.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 +#PyHive==0.6.1 +#PyInstaller==3.3.1 python-ldap==3.2.0 #python-jenkins==0.4.13 # pulled in automatically by snakebite[kerberos] From 0c80c0466e8995426abe99ce2df716568ad4bc3a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 17:28:56 +0000 Subject: [PATCH 0465/2779] added hive_schemas_csv.py --- hive_schemas_csv.py | 134 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100755 hive_schemas_csv.py diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py new file mode 100755 index 000000000..7e15501f4 --- /dev/null +++ b/hive_schemas_csv.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to Impala or HiveServer2 and dump all the schemas, tables and columns out in CSV format to stdout + +Tested on Hive 1.1.0 (CDH 5.10) with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import socket +import sys +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'HIVE_HOST', + 'HIVESERVER2_HOST', + 'IMPALA_HOST', + 'HOST' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()), + help='HiveServer2 / Impala host') + parser.add_argument('-P', '--port', type=int, default=os.getenv('PORT', 10000), + help='HiveServer2 port (default: 10000, set to 21050 for Impala)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: hive, set to \'impala\' for Impala)') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + conn = connect_db(args, None) + + if args.verbose: + log.setLevel(logging.INFO) + + print('database,table,column,type') + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + log.info('describing table %s', table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use {}'.format(database)) + column_cursor.execute('describe {}'.format(table)) + for column_row in column_cursor: + column = column_row[0] + column_type = column_row[1] + print('{database},{table},{column},{type}'\ + .format(database=database, + table=table, + column=column, + type=column_type)) + + +if __name__ == '__main__': + main() From a0c658bc62a133bd1d8ee44baf08a4da3165cd91 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 17:29:08 +0000 Subject: [PATCH 0466/2779] added impala_schemas_csv.py --- impala_schemas_csv.py | 1 + 1 file changed, 1 insertion(+) create mode 120000 impala_schemas_csv.py diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py new file mode 120000 index 000000000..baed3c963 --- /dev/null +++ b/impala_schemas_csv.py @@ -0,0 +1 @@ +hive_schemas_csv.py \ No newline at end of file From c2098d61f7f815874b4b9c4e209da94a5a663d03 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 17:40:25 +0000 Subject: [PATCH 0467/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 7e15501f4..c52fa20ee 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -45,12 +45,19 @@ log = logging.getLogger(os.path.basename(sys.argv[0])) host_envs = [ - 'HIVE_HOST', 'HIVESERVER2_HOST', + 'HIVE_HOST', 'IMPALA_HOST', 'HOST' ] +port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'IMPALA_PORT', + 'PORT' +] + def getenvs(keys, default=None): for key in keys: value = os.getenv(key) @@ -60,13 +67,15 @@ def getenvs(keys, default=None): def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()), - help='HiveServer2 / Impala host') - parser.add_argument('-P', '--port', type=int, default=os.getenv('PORT', 10000), - help='HiveServer2 port (default: 10000, set to 21050 for Impala)') + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='HiveServer2 / Impala host ' + \ + '(default: fqdn of local host, [$' + ', $'.join(host_envs) + ']') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), + help='HiveServer2 / Impala port (default: 10000, set to 21050 for Impala, [$' + \ + ', $'.join(port_envs) + ']') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') parser.add_argument('-n', '--krb5-service-name', default='hive', - help='Service principal (default: hive, set to \'impala\' for Impala)') + help='Service principal (default: \'hive\', set to \'impala\' for Impala)') parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() From 804ab385402578c3bca0c567a229a2cd5b1551e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 17:43:58 +0000 Subject: [PATCH 0468/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index c52fa20ee..7f662351a 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -18,7 +18,7 @@ Connect to Impala or HiveServer2 and dump all the schemas, tables and columns out in CSV format to stdout -Tested on Hive 1.1.0 (CDH 5.10) with Kerberos +Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 1a6125a1cf35c6ef3f69890b06b3473259e6fbde Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 17:46:56 +0000 Subject: [PATCH 0469/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 7f662351a..8b5222c73 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -18,6 +18,11 @@ Connect to Impala or HiveServer2 and dump all the schemas, tables and columns out in CSV format to stdout +CSV format: + +database,table,column,type + + Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From ce7d2dea343fc4d6cc3248e614896e5e44853c82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 17:48:14 +0000 Subject: [PATCH 0470/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 270289919..bd5bdaa1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ dicttoxml==1.7.4 GitPython==2.1.14 happybase==1.0.0 humanize==0.5.1 -#impyla==0.16.0 +impyla==0.16.0 Jinja2==2.10.1 #kazoo==2.2.1 ldif3==3.2.2 From 2c54a05bd7e10138bf68f3f0480d57668cd5867c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 17:50:10 +0000 Subject: [PATCH 0471/2779] updated help.sh --- tests/help.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/help.sh b/tests/help.sh index efac9efef..8ab17476b 100755 --- a/tests/help.sh +++ b/tests/help.sh @@ -19,13 +19,15 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh -for x in $(echo *.py 2>/dev/null); do +for x in $(echo ./*.py 2>/dev/null); do isExcluded "$x" && continue + [[ "$x" =~ (hive|impala)_schema_csv.py ]] && continue set +e - echo ./$x --help - ./$x --help # >/dev/null + echo "./$x --help" + "./$x" --help # >/dev/null status=$? set -e echo; hr From e562a9c447753a07d1a0d4555221ea32d96a06d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 19:35:44 +0000 Subject: [PATCH 0472/2779] updated help.sh --- tests/help.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/help.sh b/tests/help.sh index 8ab17476b..092820155 100755 --- a/tests/help.sh +++ b/tests/help.sh @@ -24,15 +24,15 @@ cd "$srcdir/.."; for x in $(echo ./*.py 2>/dev/null); do isExcluded "$x" && continue - [[ "$x" =~ (hive|impala)_schema_csv.py ]] && continue set +e - echo "./$x --help" - "./$x" --help # >/dev/null + echo "$x --help" + "$x" --help # >/dev/null status=$? set -e echo; hr if [ $status = 0 ]; then - [[ $x =~ ambari_blueprints ]] && continue + [[ "$x" =~ ambari_blueprints.py$ ]] && continue + [[ "$x" =~ (hive|impala)_schemas_csv.py$ ]] && continue fi [ $status = 3 ] || { echo "status code for $x --help was $status not expected 3"; exit 1; } done From 89ff0d377c3e42784f3ac952f79eab1fd42f1a5e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 19:37:02 +0000 Subject: [PATCH 0473/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 8b5222c73..4c9a11ed7 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -71,7 +71,7 @@ def getenvs(keys, default=None): return default def parse_args(): - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(description="Dumps all Hive / Impala schemas, tables, columns and types to CSV format on stdout") parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ help='HiveServer2 / Impala host ' + \ '(default: fqdn of local host, [$' + ', $'.join(host_envs) + ']') From c82af48d72b6f9e0669e2352c1444e1fa83a7863 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 19:40:44 +0000 Subject: [PATCH 0474/2779] updated excluded.sh --- tests/excluded.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/excluded.sh b/tests/excluded.sh index b02a8c6bb..68f62c58f 100755 --- a/tests/excluded.sh +++ b/tests/excluded.sh @@ -23,8 +23,8 @@ set -eu isExcluded(){ local prog="$1" [[ "$prog" =~ ^\* ]] && return 0 + [[ "$prog" =~ spark_.*.py ]] && return 0 #[[ $prog =~ ipython-notebook ]] && return 0 - #[[ $prog =~ ambari_blueprints.py ]] && return 0 # this external git check is expensive, skip it when in CI as using fresh git checkouts is_CI && return 1 # we no longer include spark inside pytools docker image due to bloat, so skip tests From a3ab248856bd8800169c4d8b6c398ab6339e79fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Nov 2019 19:44:57 +0000 Subject: [PATCH 0475/2779] updated help.sh --- tests/help.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/help.sh b/tests/help.sh index 092820155..851451152 100755 --- a/tests/help.sh +++ b/tests/help.sh @@ -33,6 +33,7 @@ for x in $(echo ./*.py 2>/dev/null); do if [ $status = 0 ]; then [[ "$x" =~ ambari_blueprints.py$ ]] && continue [[ "$x" =~ (hive|impala)_schemas_csv.py$ ]] && continue + [[ "$x" =~ pythonpath.py$ ]] && continue fi [ $status = 3 ] || { echo "status code for $x --help was $status not expected 3"; exit 1; } done From eff7903447f5d889a562225d993e41c763034444 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Nov 2019 09:47:49 +0000 Subject: [PATCH 0476/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index c15564a57..a0015d809 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -14,6 +14,7 @@ # ============================================================================ # gcc-c++ +# needed to build python-krbV and cloudera/thrift_sasl cyrus-sasl-devel krb5-devel openssl-devel From 9fa5c40c5764fd51401308f7d443fea8506838e2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Nov 2019 09:47:51 +0000 Subject: [PATCH 0477/2779] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index 9e9899331..a2e311a6b 100644 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -17,5 +17,5 @@ libldap2-dev # needed to build python-snappy for avro module libsnappy-dev -# needed to build python-krbV +# needed to build python-krbV and cloudera/thrift_sasl libsasl2-dev From 89a1315e4fd117b9d3d414e9a3653e70900b7bdd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Nov 2019 09:49:15 +0000 Subject: [PATCH 0478/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 4c9a11ed7..6af9afe85 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -71,7 +71,8 @@ def getenvs(keys, default=None): return default def parse_args(): - parser = argparse.ArgumentParser(description="Dumps all Hive / Impala schemas, tables, columns and types to CSV format on stdout") + parser = argparse.ArgumentParser( + description="Dumps all Hive / Impala schemas, tables, columns and types to CSV format on stdout") parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ help='HiveServer2 / Impala host ' + \ '(default: fqdn of local host, [$' + ', $'.join(host_envs) + ']') From 7b19dd4d60362dd8a9b1f7951287bb674aee2dca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Nov 2019 10:41:43 +0000 Subject: [PATCH 0479/2779] set port defaults when called as impala --- hive_schemas_csv.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 6af9afe85..7f1c7f188 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -44,7 +44,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -75,16 +75,28 @@ def parse_args(): description="Dumps all Hive / Impala schemas, tables, columns and types to CSV format on stdout") parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ help='HiveServer2 / Impala host ' + \ - '(default: fqdn of local host, [$' + ', $'.join(host_envs) + ']') + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), - help='HiveServer2 / Impala port (default: 10000, set to 21050 for Impala, [$' + \ - ', $'.join(port_envs) + ']') + help='HiveServer2 / Impala port (default: 10000 if called as hive, ' + \ + '21050 if called as impala, $' + \ + ', $'.join(port_envs) + ')') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') parser.add_argument('-n', '--krb5-service-name', default='hive', - help='Service principal (default: \'hive\', set to \'impala\' for Impala)') + help='Service principal (default: \'hive\', or \'impala\' if called as impala_schema_csv.py)') parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + + if 'impala' in sys.argv[0]: + if args.krb5_service_name == 'hive': + log.info('called as impala, setting service principal to impala') + args.krb5_service_name = 'impala' + if args.port == 10000: + log.info('called as impala, setting port to 21050') + args.port = 21050 return args def connect_db(args, database): @@ -109,9 +121,6 @@ def main(): conn = connect_db(args, None) - if args.verbose: - log.setLevel(logging.INFO) - print('database,table,column,type') log.info('querying databases') with conn.cursor() as db_cursor: From 7349f9cad1cf7fa2931b73906ce8a176cf0abd9b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Nov 2019 10:44:28 +0000 Subject: [PATCH 0480/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 7f1c7f188..b5e872c54 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -82,7 +82,7 @@ def parse_args(): ', $'.join(port_envs) + ')') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') parser.add_argument('-n', '--krb5-service-name', default='hive', - help='Service principal (default: \'hive\', or \'impala\' if called as impala_schema_csv.py)') + help='Service principal (default: \'hive\', or \'impala\' if called as impala_schemas_csv.py)') parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() From 2e43e6a3d9d7f966837068a3005d6bf1713ff9d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Nov 2019 10:47:05 +0000 Subject: [PATCH 0481/2779] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 66d9791a6..27cd69ba9 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ Environment variables are supported for convenience and also to hide credentials - ```hadoop_hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - ```hadoop_hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): + - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From c76ffe488525581fdcd84928628d4a0ac7974729 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Nov 2019 11:15:40 +0000 Subject: [PATCH 0482/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index b5e872c54..925fb6ef9 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -16,7 +16,15 @@ """ -Connect to Impala or HiveServer2 and dump all the schemas, tables and columns out in CSV format to stdout +Connect to a HiveServer2 or Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout + +In practice Hive is much more reliable for dumping masses of schema + +Impala appears faster initially but then slows down more than Hive and hits things query handle errors +under sustained load of extracting large amounts of schema information + +There is also a risk that Impala's metadata may be out of date, so Hive is strongly preferred for this + CSV format: From a9f5cefa3b40df9e659692edc3b56cc8fe7747ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Nov 2019 17:42:53 +0000 Subject: [PATCH 0483/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6c486a768..f77f6f741 100755 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ python: @# only install pip packages not installed via system packages @#$(SUDO_PIP) pip install --upgrade -r requirements.txt @#$(SUDO_PIP) pip install -r requirements.txt - @bash-tools/python_pip_install_if_absent.sh requirements.txt + @PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt @# python-krbV dependency doesn't build on Mac any more and is unmaintained @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit From 1d8cc35177564be0217961e75f5dc95b86e4b594 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 11 Nov 2019 11:19:15 +0000 Subject: [PATCH 0484/2779] added quoting to work around decimal(x,y) data types --- hive_schemas_csv.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 925fb6ef9..82b1ab320 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -31,6 +31,11 @@ database,table,column,type +I recommend generating quoted csv because you may encounter Hive data types such as decimal(15,2) +which would cause incorrect field splitting, you can disable by setting --quotechar='' to blank but +if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will +raise a traceback to tell you to set one (eg. --escapechar='\\') + Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see @@ -45,6 +50,7 @@ from __future__ import unicode_literals import argparse +import csv import logging import os import socket @@ -52,7 +58,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.1' +__version__ = '0.2.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -92,6 +98,12 @@ def parse_args(): parser.add_argument('-n', '--krb5-service-name', default='hive', help='Service principal (default: \'hive\', or \'impala\' if called as impala_schemas_csv.py)') parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + # must set type to str otherwise csv module gives this error on Python 2.7: + # TypeError: "delimiter" must be string, not unicode + parser.add_argument('-d', '--delimiter', default=',', type=str, help='Delimiter to use (default: ,)') + parser.add_argument('-Q', '--quotechar', default='"', type=str, + help='Generate quoted CSV (recommended, default is double quote \'"\')') + parser.add_argument('-E', '--escapechar', help='Escape char if needed') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() @@ -129,7 +141,17 @@ def main(): conn = connect_db(args, None) - print('database,table,column,type') + quoting = csv.QUOTE_ALL + if args.quotechar == '': + quoting = csv.QUOTE_NONE + fieldnames = ['database', 'table', 'column', 'type'] + csv_writer = csv.DictWriter(sys.stdout, + delimiter=args.delimiter, + quotechar=args.quotechar, + escapechar=args.escapechar, + quoting=quoting, + fieldnames=fieldnames) + csv_writer.writeheader() log.info('querying databases') with conn.cursor() as db_cursor: db_cursor.execute('show databases') @@ -155,11 +177,10 @@ def main(): for column_row in column_cursor: column = column_row[0] column_type = column_row[1] - print('{database},{table},{column},{type}'\ - .format(database=database, - table=table, - column=column, - type=column_type)) + csv_writer.writerow({'database': database, + 'table': table, + 'column': column, + 'type': column_type}) if __name__ == '__main__': From ede4c8d9775f2bc5572b4b5e2c2b33b2c9eb760d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 10:57:19 +0000 Subject: [PATCH 0485/2779] added hive_compute_table_stats.py, impala_compute_table_stats.py --- hive_compute_table_stats.py | 172 ++++++++++++++++++++++++++++++++++ impala_compute_table_stats.py | 170 +++++++++++++++++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100755 hive_compute_table_stats.py create mode 100755 impala_compute_table_stats.py diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py new file mode 100755 index 000000000..59f13d2d5 --- /dev/null +++ b/hive_compute_table_stats.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to a HiveServer2 and compute optimization statistics on all tables, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Hive 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' +] + +port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Computes statistics on all Hive tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='HiveServer2 host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), + help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + conn = connect_db(args, None) + + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + compute_table_stats(conn, args, database, table, partition_regex) + +def compute_table_stats(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + partitions_found = False + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {}'.format(database)) + partition_cursor.execute('show partitions {}'.format(database)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + partitions_found = True + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('ANALYZE TABLE {db}.{table} PARTITION({key}={value}) COMPUTE STATISTICS'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + if not partitions_found: + log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) + with conn.cursor() as table_cursor: + log.info("running compute stats on table '%s'", table) + table_cursor.execute('ANALYZE TABLE {db}.{table} COMPUTE STATISTICS'.format(db=database, table=table)) + + +if __name__ == '__main__': + main() diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py new file mode 100755 index 000000000..669959f49 --- /dev/null +++ b/impala_compute_table_stats.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to a Impalad and compute optimization statistics on all tables, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Impala 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'IMPALA_HOST', + 'HOST' +] + +port_envs = [ + 'IMPALA_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Computes statistics on all Impala tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='Impalad host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), + help='Impalad port (default: 21050, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + conn = connect_db(args, None) + + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + compute_table_stats(conn, args, database, table, partition_regex) + +def compute_table_stats(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + partitions_found = False + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {}'.format(database)) + partition_cursor.execute('show partitions {}'.format(database)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + partitions_found = True + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('COMPUTE INCREMENTAL STATS {db}.{table} PARTITION({key}={value})'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + if not partitions_found: + log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) + with conn.cursor() as table_cursor: + log.info("running compute stats on table '%s'", table) + table_cursor.execute('COMPUTE STATS {db}.{table}'.format(db=database, table=table)) + + +if __name__ == '__main__': + main() From a57ad82821d24572d41b49bc08178b003dfc42d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 11:50:13 +0000 Subject: [PATCH 0486/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cea4c6fae..6b034a1ac 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cea4c6fae729d43015eb861fbf52f03abe52f2c0 +Subproject commit 6b034a1acb99f5d1686a496fec900c306e2874ce From 28635feb83f6a04671871b74800590acae3b8680 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 11:50:36 +0000 Subject: [PATCH 0487/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6b034a1ac..af1b3c13d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6b034a1acb99f5d1686a496fec900c306e2874ce +Subproject commit af1b3c13d4a3cdf0897284ab31e19ad5b85044f2 From 585c2b2584cbb0dd0df24b09ab0c7ecb4856a833 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 11:51:29 +0000 Subject: [PATCH 0488/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 93cc037e0..3dbf482b0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 93cc037e06a8f4c360369e0e76a0550dc2822976 +Subproject commit 3dbf482b086e301d454511111c884814ce84acd2 From dcf4c49a7b628086e9fcfc5f407f436e668fbab9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 14:49:57 +0000 Subject: [PATCH 0489/2779] updated hive_compute_table_stats.py --- hive_compute_table_stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py index 59f13d2d5..1dd33b885 100755 --- a/hive_compute_table_stats.py +++ b/hive_compute_table_stats.py @@ -165,6 +165,7 @@ def compute_table_stats(conn, args, database, table, partition_regex): log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) with conn.cursor() as table_cursor: log.info("running compute stats on table '%s'", table) + # doesn't support parameterized query quoting from dbapi spec table_cursor.execute('ANALYZE TABLE {db}.{table} COMPUTE STATISTICS'.format(db=database, table=table)) From 4c3dfef1572dedabf07f4c19d8c108f7efc35fde Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 14:50:05 +0000 Subject: [PATCH 0490/2779] updated impala_compute_table_stats.py --- impala_compute_table_stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py index 669959f49..f545224bc 100755 --- a/impala_compute_table_stats.py +++ b/impala_compute_table_stats.py @@ -163,6 +163,7 @@ def compute_table_stats(conn, args, database, table, partition_regex): log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) with conn.cursor() as table_cursor: log.info("running compute stats on table '%s'", table) + # doesn't support parameterized query quoting from dbapi spec table_cursor.execute('COMPUTE STATS {db}.{table}'.format(db=database, table=table)) From ba23262fcb2d6e82734ae2fd5cfb924f64c63db6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 17:13:22 +0000 Subject: [PATCH 0491/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index af1b3c13d..545423ba2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit af1b3c13d4a3cdf0897284ab31e19ad5b85044f2 +Subproject commit 545423ba200b85def8c854315f1cb1d8f119ab24 From b77d9b65aece0a2ff3e276990a9c379700327980 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 Nov 2019 17:13:22 +0000 Subject: [PATCH 0492/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3dbf482b0..dc3c170cf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3dbf482b086e301d454511111c884814ce84acd2 +Subproject commit dc3c170cf80d4a41fdc2797fcc5316247eda920c From 13d81bb707adc1208bdc7cfa44492819691dbbde Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Nov 2019 10:06:52 +0000 Subject: [PATCH 0493/2779] added regex try/except --- hive_compute_table_stats.py | 12 ++++++++---- impala_compute_table_stats.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py index 1dd33b885..0306c44ae 100755 --- a/hive_compute_table_stats.py +++ b/hive_compute_table_stats.py @@ -108,11 +108,15 @@ def connect_db(args, database): def main(): args = parse_args() - conn = connect_db(args, None) + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - partition_regex = re.compile(args.partition, re.I) + conn = connect_db(args, None) log.info('querying databases') with conn.cursor() as db_cursor: diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py index f545224bc..ddfaeeba8 100755 --- a/impala_compute_table_stats.py +++ b/impala_compute_table_stats.py @@ -106,11 +106,15 @@ def connect_db(args, database): def main(): args = parse_args() - conn = connect_db(args, None) + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - partition_regex = re.compile(args.partition, re.I) + conn = connect_db(args, None) log.info('querying databases') with conn.cursor() as db_cursor: From 4710498a37f28baa447cddf36c3a6a6382126936 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Nov 2019 23:35:14 +0000 Subject: [PATCH 0494/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cea4c6fae..1b52566af 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cea4c6fae729d43015eb861fbf52f03abe52f2c0 +Subproject commit 1b52566af83b38e372cbf0176fd3420e6bd7ff4e From efcebf203403da280b78958a26d7dd064e2fb356 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Nov 2019 23:35:32 +0000 Subject: [PATCH 0495/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 93cc037e0..912ea1b62 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 93cc037e06a8f4c360369e0e76a0550dc2822976 +Subproject commit 912ea1b622e0056bc4f51572dcf795e32a5ad097 From a295f828dd0837d7f6ca086bdc92d2dbcfefd8a0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:32:54 +0000 Subject: [PATCH 0496/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b52566af..aa71f5453 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b52566af83b38e372cbf0176fd3420e6bd7ff4e +Subproject commit aa71f5453337b4404fb2c2bd082f506d7fcb7bd3 From c9bf73567f398d2f8a6c8fbba2ddbf9a75bffe32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:32:54 +0000 Subject: [PATCH 0497/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 912ea1b62..42a3563d6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 912ea1b622e0056bc4f51572dcf795e32a5ad097 +Subproject commit 42a3563d6277348774d6839e282672824ae694f1 From a17615b59072a0a0fa04f230fa44266a7ee67427 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:42:03 +0000 Subject: [PATCH 0498/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index aa71f5453..d6221cd31 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit aa71f5453337b4404fb2c2bd082f506d7fcb7bd3 +Subproject commit d6221cd31b65db515b0b755db3742e5c874f7a6d From 68d037e90671313adc0f3fbdd5a7b830adaadea2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:42:03 +0000 Subject: [PATCH 0499/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 42a3563d6..f46398a33 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 42a3563d6277348774d6839e282672824ae694f1 +Subproject commit f46398a3372530c148e1fdc760b2a2659618aa3c From bf226d32d93dafd1c1d0d56de7129bce2b62e79c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:43:54 +0000 Subject: [PATCH 0500/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d6221cd31..c6a900361 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d6221cd31b65db515b0b755db3742e5c874f7a6d +Subproject commit c6a9003610ddd2321dc28c2c7d69f98eb43ba7c7 From 8ac30eab41190918f9f063b3065edba2f69edfe5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Nov 2019 00:43:55 +0000 Subject: [PATCH 0501/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f46398a33..cf0435d86 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f46398a3372530c148e1fdc760b2a2659618aa3c +Subproject commit cf0435d8687fc0951c91ed5cb410a84bce31aca0 From bc0aea723c233b2fae0005d3f06044db1aad92c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 1 Dec 2019 21:53:10 +0000 Subject: [PATCH 0502/2779] added exception to not take Original Amount foreign currency column --- crunch_accounting_csv_statement_converter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 34069c12b..477198dbf 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.2' +__version__ = '0.6.3' class CrunchAccountingCsvStatementConverter(CLI): @@ -169,7 +169,8 @@ def detect_columns(self, csvreader): positions['date'] = position elif 'Merchant Name' in value: positions['desc'] = position - elif 'Amount' in value: + # Original Amount column will be original currency eg 499 USD, but we only want native currency eg. 421.33 + elif 'Amount' in value and not 'Original' in value: positions['amount'] = position elif 'Balance' in value: balance_position = position From e5d5f70bd8a9b9b593710dea8f46851b2276cbf1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Dec 2019 16:25:10 +0000 Subject: [PATCH 0503/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 82b1ab320..3058e28dc 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -42,6 +42,15 @@ https://github.com/cloudera/impyla/issues/286 +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + """ from __future__ import absolute_import @@ -58,7 +67,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.2.1' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -104,11 +113,13 @@ def parse_args(): parser.add_argument('-Q', '--quotechar', default='"', type=str, help='Generate quoted CSV (recommended, default is double quote \'"\')') parser.add_argument('-E', '--escapechar', help='Escape char if needed') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + parser.add_argument('-v', '--verbose', action='count', help='Verbose mode') args = parser.parse_args() if args.verbose: log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) if 'impala' in sys.argv[0]: if args.krb5_service_name == 'hive': @@ -123,6 +134,10 @@ def connect_db(args, database): auth_mechanism = None if args.kerberos: auth_mechanism = 'GSSAPI' + log.debug('kerberos enabled') + log.debug('krb5 remote service principal name = %s', args.krb5_service_name) + if args.ssl is True: + log.debug('ssl enabled') log.info('connecting to %s:%s database %s', args.host, args.port, database) return connect( @@ -139,7 +154,7 @@ def connect_db(args, database): def main(): args = parse_args() - conn = connect_db(args, None) + conn = connect_db(args, 'default') quoting = csv.QUOTE_ALL if args.quotechar == '': From 035a49b6be15b0cc1412117bc9e6a00c1c40f928 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 17:14:31 +0000 Subject: [PATCH 0504/2779] updated ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index b951f3244..4310b15a6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,7 +1,3 @@ -Please be as specific as possible when raising an issue. - -- what were you expecting -- what was the result -- what was the output - if running a CLI program please include the full debug output when running with program by using the `--debug` or `-v -v -v` switches for most tools, or setting `export DEBUG=1` environment variable, which also works with the shell scripts. +Please be specific about your issue and include debug output from running after setting `export DEBUG=1` in your shell. You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively. From bb9dc4bd0f94c2f1c9e186be2aafc26502b2083a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 17:15:22 +0000 Subject: [PATCH 0505/2779] updated ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 4310b15a6..5f04c0a58 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,3 +1,3 @@ -Please be specific about your issue and include debug output from running after setting `export DEBUG=1` in your shell. +Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell. You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively. From 8497d24fb64bbe29df167a87f77af48485d06448 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 17:16:31 +0000 Subject: [PATCH 0506/2779] updated validate_yaml.py --- validate_yaml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/validate_yaml.py b/validate_yaml.py index 2852a1905..285f75ef8 100755 --- a/validate_yaml.py +++ b/validate_yaml.py @@ -49,7 +49,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.1' +__version__ = '0.9.2' class YamlValidatorTool(CLI): @@ -99,7 +99,8 @@ def check_yaml(self, content): if not self.get_opt('print'): if self.verbose > 2: try: - yaml.safe_load(content) + # TODO: doesn't validate network-policy.yaml in templates containing multiple yaml docs + yaml.safe_load_all(content) except yaml.YAMLError as _: print(_) die(self.invalid_yaml_msg) From 124cfc421e7d2329894a0ae53b5e07cc3d512270 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 18:09:18 +0000 Subject: [PATCH 0507/2779] added aws_users_access_key_age.py --- aws_users_access_key_age.py | 109 ++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 aws_users_access_key_age.py diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py new file mode 100755 index 000000000..1a65f1085 --- /dev/null +++ b/aws_users_access_key_age.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-13 17:24:40 +0000 (Fri, 13 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Lists all AWS IAM users keys along with their ages, optionally filtering any older than a given number of days + +Output format is: + + + +Status is usually Active + +See also aws_users_access_key_age.sh for a similar version in the adjacent DevOps Bash Tools repo +This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +import datetime +import boto3 +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_float + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +class AWSUsersAccessKeysAge(CLI): + + def __init__(self): + super(AWSUsersAccessKeysAge, self).__init__() + self.age = None + self.only_active_keys = False + + def add_options(self): + self.add_opt('--age', help='Return keys older than this N days') + self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') + + def process_args(self): + self.only_active_keys = self.get_opt('only_active') + self.age = self.get_opt('age') + if self.age: + validate_float(self.age, 'age') + self.age = float(self.age) + + def run(self): + iam = boto3.client('iam') + user_paginator = iam.get_paginator('list_users') + now = datetime.datetime.utcnow() + age = 0 + if self.age: + age = self.age * 86400 + for users_response in user_paginator.paginate(): + for user_item in users_response['Users']: + username = user_item['UserName'] + key_paginator = iam.get_paginator('list_access_keys') + for keys_response in key_paginator.paginate(UserName=username): + self.process_key(keys_response, username, now, age) + log.info('Completed') + + def process_key(self, keys_response, username, now, age): + assert not keys_response['IsTruncated'] + for access_key_item in keys_response['AccessKeyMetadata']: + assert username == access_key_item['UserName'] + status = access_key_item['Status'] + if self.only_active_keys and not status: + continue + create_date = access_key_item['CreateDate'] + #if age: + # already cast to datetime.datetime + #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') + #if (now - create_date).total_seconds() < age: + # continue + print('{user:20}\t{status}\t{date}'.format( + user=username, + status=status, + date=create_date)) + + +if __name__ == '__main__': + AWSUsersAccessKeysAge().main() From 17fc67218a7fcba43d75a6c1f91aac28e02eb0fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 18:09:45 +0000 Subject: [PATCH 0508/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 1a65f1085..4705025d8 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -62,11 +62,11 @@ def __init__(self): def add_options(self): self.add_opt('--age', help='Return keys older than this N days') - self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') + #self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') def process_args(self): self.only_active_keys = self.get_opt('only_active') - self.age = self.get_opt('age') + #self.age = self.get_opt('age') if self.age: validate_float(self.age, 'age') self.age = float(self.age) From 17567fafe31399399060fe6e1a46d381ef476482 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Dec 2019 18:10:13 +0000 Subject: [PATCH 0509/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 4705025d8..41517a60d 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -61,8 +61,8 @@ def __init__(self): self.only_active_keys = False def add_options(self): - self.add_opt('--age', help='Return keys older than this N days') - #self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') + #self.add_opt('--age', help='Return keys older than this N days') + self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') def process_args(self): self.only_active_keys = self.get_opt('only_active') From f7bd965d0ad1bd74f4c3c8aad74b83c25bc417b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:00:47 +0000 Subject: [PATCH 0510/2779] added --age support to report only keys older than N days --- aws_users_access_key_age.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 41517a60d..9ccfd543d 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -51,42 +51,41 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' class AWSUsersAccessKeysAge(CLI): def __init__(self): super(AWSUsersAccessKeysAge, self).__init__() self.age = None + self.now = None self.only_active_keys = False def add_options(self): - #self.add_opt('--age', help='Return keys older than this N days') + self.add_opt('--age', help='Return keys older than N days') self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') def process_args(self): self.only_active_keys = self.get_opt('only_active') - #self.age = self.get_opt('age') + self.age = self.get_opt('age') if self.age: validate_float(self.age, 'age') self.age = float(self.age) + self.age = self.age * 86400 def run(self): iam = boto3.client('iam') user_paginator = iam.get_paginator('list_users') - now = datetime.datetime.utcnow() - age = 0 - if self.age: - age = self.age * 86400 + self.now = datetime.datetime.utcnow() for users_response in user_paginator.paginate(): for user_item in users_response['Users']: username = user_item['UserName'] key_paginator = iam.get_paginator('list_access_keys') for keys_response in key_paginator.paginate(UserName=username): - self.process_key(keys_response, username, now, age) + self.process_key(keys_response, username) log.info('Completed') - def process_key(self, keys_response, username, now, age): + def process_key(self, keys_response, username): assert not keys_response['IsTruncated'] for access_key_item in keys_response['AccessKeyMetadata']: assert username == access_key_item['UserName'] @@ -94,11 +93,14 @@ def process_key(self, keys_response, username, now, age): if self.only_active_keys and not status: continue create_date = access_key_item['CreateDate'] - #if age: - # already cast to datetime.datetime + if self.age: + # already cast to datetime.datetime with tzinfo #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') - #if (now - create_date).total_seconds() < age: - # continue + # removing tzinfo for comparison to avoid below error + # - both are UTC and this doesn't make much difference anyway + # TypeError: can't subtract offset-naive and offset-aware datetimes + if (self.now - create_date.replace(tzinfo=None)).total_seconds() < self.age: + continue print('{user:20}\t{status}\t{date}'.format( user=username, status=status, From 18ef21ff3fe6b339bc5adefaa5a725394e4660f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:00:54 +0000 Subject: [PATCH 0511/2779] updated requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index bd5bdaa1c..77613f9cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ avro==1.8.1 #avro-python3==1.9.0 awscli==1.16.241 #bitarray==0.8.1 +#boto==2.49.0 +boto3==1.10.37 #cassandra-driver==3.6.0 dicttoxml==1.7.4 # Elasticsearch library must match major version :( From 08ab4ccce4b080e021a71320da3ebf061389681a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:41:12 +0000 Subject: [PATCH 0512/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 9ccfd543d..7523e7947 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -86,7 +86,7 @@ def run(self): log.info('Completed') def process_key(self, keys_response, username): - assert not keys_response['IsTruncated'] + #assert not keys_response['IsTruncated'] for access_key_item in keys_response['AccessKeyMetadata']: assert username == access_key_item['UserName'] status = access_key_item['Status'] From 855332eac4f957dd1509939aa03be1efb3825ca0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:42:40 +0000 Subject: [PATCH 0513/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 7523e7947..8116e21e6 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -34,9 +34,9 @@ from __future__ import print_function from __future__ import unicode_literals +import datetime import os import sys -import datetime import boto3 libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) From e42fe1ec4e1b7892fae8166b372478b5873c27a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 11:58:11 +0000 Subject: [PATCH 0514/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 8116e21e6..b107fdcb3 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -90,7 +90,7 @@ def process_key(self, keys_response, username): for access_key_item in keys_response['AccessKeyMetadata']: assert username == access_key_item['UserName'] status = access_key_item['Status'] - if self.only_active_keys and not status: + if self.only_active_keys and status != 'Active': continue create_date = access_key_item['CreateDate'] if self.age: From 4c0aa16586980d9dea90e3f9cc9ac8e2c0e049e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 13:55:15 +0000 Subject: [PATCH 0515/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index b107fdcb3..aab16bd38 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -62,8 +62,8 @@ def __init__(self): self.only_active_keys = False def add_options(self): - self.add_opt('--age', help='Return keys older than N days') - self.add_opt('--only-active', action='store_true', help='Return only keys with Active status') + self.add_opt('-a', '--age', help='Return keys older than N days') + self.add_opt('-o', '--only-active', action='store_true', help='Return only keys with Active status') def process_args(self): self.only_active_keys = self.get_opt('only_active') From a0887a072afcac0128c3df64a5876ad17950b0d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Dec 2019 17:07:53 +0000 Subject: [PATCH 0516/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index aab16bd38..da6ca7509 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -24,9 +24,16 @@ Status is usually Active -See also aws_users_access_key_age.sh for a similar version in the adjacent DevOps Bash Tools repo +See also: + +aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo (https://github.com/harisekhon/devops-bash-tools) This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies +Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins) + +check_aws_access_keys_age.py +check_aws_access_keys_disabled.py + """ from __future__ import absolute_import From 64dc96d6b065de477aae022ecf87325df1b12d8a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Dec 2019 10:05:43 +0000 Subject: [PATCH 0517/2779] set timeout higher --- aws_users_access_key_age.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index da6ca7509..eedbcad26 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -26,7 +26,9 @@ See also: -aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo (https://github.com/harisekhon/devops-bash-tools) +aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo +- https://github.com/harisekhon/devops-bash-tools + This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins) @@ -67,6 +69,7 @@ def __init__(self): self.age = None self.now = None self.only_active_keys = False + self.timeout_default = 300 def add_options(self): self.add_opt('-a', '--age', help='Return keys older than N days') From 7e549bbff0e9d15ee5acc7b86b40d2aaca6646b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Dec 2019 11:47:37 +0000 Subject: [PATCH 0518/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index eedbcad26..72b13fa7f 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -24,17 +24,21 @@ Status is usually Active -See also: +Uses Boto, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html -aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo -- https://github.com/harisekhon/devops-bash-tools +See also: This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies + aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo + - https://github.com/harisekhon/devops-bash-tools + Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins) -check_aws_access_keys_age.py -check_aws_access_keys_disabled.py + check_aws_access_keys_age.py + check_aws_access_keys_disabled.py """ From ddffcc39ebbc9d2d421d119f2ace11e43c95313f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 10:06:17 +0000 Subject: [PATCH 0519/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 72b13fa7f..0d11e6616 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -22,7 +22,7 @@ -Status is usually Active +Status is Active or Inactive Uses Boto, read here for the list of ways to configure your AWS credentials: @@ -115,7 +115,7 @@ def process_key(self, keys_response, username): # TypeError: can't subtract offset-naive and offset-aware datetimes if (self.now - create_date.replace(tzinfo=None)).total_seconds() < self.age: continue - print('{user:20}\t{status}\t{date}'.format( + print('{user:20}\t{status:8}\t{date}'.format( user=username, status=status, date=create_date)) From d4eff1c534234f9f60d1514c0bf4b76ed87194b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 10:41:20 +0000 Subject: [PATCH 0520/2779] added support for AWS user/group arns --- anonymize.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index 74cb19226..7a069e41f 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.5' +__version__ = '0.9.6' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -297,6 +297,8 @@ def __init__(self): 'group2': r'({group_name}{sep}){user}'.format(group_name=group_name, sep=arg_sep, user=user_regex), 'group3': r'for\s+group\s+{group}'.format(group=user_regex), 'group4': r'(["\']{group_name}["\']\s*:\s*["\']?){group}'.format(group_name=group_name, group=user_regex), + 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format(\ + group='({}/)*{}'.format(user_regex, user_regex)), 'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/(home|user)/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), @@ -307,6 +309,8 @@ def __init__(self): 'user6': r'(?/){user}@'.format(user=user_regex), 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'\ .format(user_name=user_name, user=user_regex), + #'user8': r'arn:aws:iam::\d{12}:user/{user}'.format(user=user_regex), + 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format(user='({}/)*{}'.format(user_regex, user_regex)), 'password': r'([-\.]?{pass_word_phrase}{sep}){pw}'\ .format(pass_word_phrase=pass_word_phrase, sep=arg_sep, @@ -396,10 +400,12 @@ def __init__(self): 'user5': 'for user ', 'user6': '@', 'user7': r'\1', + 'user8': r'\1\2', 'group': r'\1', 'group2': r'\1', 'group3': r'for group ', 'group4': r'\1', + 'group5': r'\1\2', 'password': r'\1', 'password2': r'\1:', 'password3': r'\1\2', @@ -792,16 +798,16 @@ def anonymize(self, line): line = self.anonymize_dynamic(_, line) if line is None: if method: - raise AssertionError('anonymize_{} returned None', _) + raise AssertionError('anonymize_{} returned None'.format(_)) else: - raise AssertionError('anonymize_dynamic({}, line)', _) + raise AssertionError('anonymize_dynamic({}, line)'.format(_)) line += line_ending return line def anonymize_dynamic(self, name, line): #log.debug('anonymize_dynamic(%s, %s)', name, line) if not isStr(line): - raise AssertionError('anonymize_dynamic: passed in non-string line: %s', line) + raise AssertionError('anonymize_dynamic: passed in non-string line: {}'.format(line)) line = self.dynamic_replace(name, line) for i in range(2, 101): name2 = '{}{}'.format(name, i) From f39382c662a5c2239f22bc0cf49a7665953ab674 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 10:41:24 +0000 Subject: [PATCH 0521/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 20ff7fa24..f30e9ac0a 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -430,6 +430,18 @@ dest[111]="127.0.0.1" src[112]="travis token: Abc123" dest[112]="travis token: " +src[113]="arn:aws:iam::123456789012:user/hari" +dest[113]="arn:aws:iam:::user/" + +src[114]="arn:aws:iam::123456789012:group/hari" +dest[114]="arn:aws:iam:::group/" + +src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*" +dest[115]="arn:aws:iam:::user//*" + +src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" +dest[116]="arn:aws:iam:::group//*" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 7842b013daf8c8a490c4df6a36caaeca07d9e525 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 11:23:31 +0000 Subject: [PATCH 0522/2779] added --aws support --- anonymize.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 7a069e41f..aae935b51 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.6' +__version__ = '0.9.7' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -114,6 +114,7 @@ def __init__(self): self.hash_salt = None # order of iteration of application matters because we must do more specific matches before less specific ones self.anonymizations = OrderedDict([ + ('aws', False), ('ip_prefix', False), ('ip', False), ('subnet_mask', False), @@ -282,6 +283,14 @@ def __init__(self): # openssl uses -passin switch pass_word_phrase = r'(?:pass(?:word|phrase|in)?|userPassword)' self.regex = { + # arn:partition:service:region:account-id:resource-id + # arn:partition:service:region:account-id:resource-type/resource-id + # arn:partition:service:region:account-id:resource-type:resource-id + # eg. arn:aws:iam::123456789012:group/Development/product_1234/* + 'aws': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d+(:([^:/]+)[:/])[\w/-]+', + # arn:aws:s3:::my_corporate_bucket/Development/* + #'aws2': r'\b(arn:aws:s3:::)[^/]+', + 'aws2': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d*:[\w/-]+', # don't change hostname or fqdn regex without updating hash_hostnames() option parse # since that replaces these replacements and needs to match the grouping captures and surrounding format 'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex), @@ -384,6 +393,11 @@ def __init__(self): ldap_lambda_lowercase = lambda m: r'{}<{}>'.format(m.group(1), m.group(2).lower()) # will auto-infer replacements to not have to be explicit, use this only for override mappings self.replacements = { + # arn:partition:service:region:account-id:resource-id + # arn:partition:service:region:account-id:resource-type/resource-id + # arn:partition:service:region:account-id:resource-type:resource-id + 'aws': r'\1\2<\3>', + 'aws2': r'\1:', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', @@ -462,6 +476,7 @@ def add_options(self): self.add_opt('-a', '--all', action='store_true', help='Apply all anonymizations (careful this includes --host which can be overzealous and ' + \ 'match too many things, in which case try more targeted anonymizations below)') + self.add_opt('-w', '--aws', action='store_true', help='Apply AWS ARN anonymizations'), self.add_opt('-C', '--custom', action='store_true', help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \ 'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \ @@ -573,6 +588,7 @@ def process_options(self): if _ in ('subnet_mask', 'mac', 'group'): continue self.anonymizations[_] = self.get_opt(_) + log.debug('anonymization enabled %s = %s', _, bool(self.anonymizations[_])) self._process_options_host() self._process_options_network() self._process_options_exceptions() From 2fb81ce44b6801685514ad55c2cebd4868eee434 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 11:23:38 +0000 Subject: [PATCH 0523/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index f30e9ac0a..fb0b8036a 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -437,10 +437,16 @@ src[114]="arn:aws:iam::123456789012:group/hari" dest[114]="arn:aws:iam:::group/" src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*" -dest[115]="arn:aws:iam:::user//*" +dest[115]="arn:aws:iam:::user/*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" -dest[116]="arn:aws:iam:::group//*" +dest[116]="arn:aws:iam:::group/*" + +src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" +dest[116]="arn:aws:iam:::group/*" + +src[117]="arn:aws:s3:::my_corporate_bucket/Development/*" +dest[117]="arn:aws:s3:::*" # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 222ec6e61cb63277ecdafb7fc3b2589bc7a72c3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Dec 2019 11:41:45 +0000 Subject: [PATCH 0524/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 27cd69ba9..760534d27 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Environment variables are supported for convenience and also to hide credentials - hostnames / domains / FQDNs - email addresses - IP + MAC addresses + - AWS ARNs - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings From 323e9c753fed21527c754bfe2643a23e21f3ca85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 09:48:57 +0000 Subject: [PATCH 0525/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 0d11e6616..a4b2e503d 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -28,11 +28,11 @@ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html -See also: - This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies - aws_users_access_key_age.sh - similar version in the adjacent DevOps Bash Tools repo +See also: + + aws_users_access_key_age.sh - simpler version in the adjacent DevOps Bash Tools repo without age filtering - https://github.com/harisekhon/devops-bash-tools Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins) From 15b96613977395f020e389e330d9b95819404e07 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 11:24:59 +0000 Subject: [PATCH 0526/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index a4b2e503d..89dbbfa02 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -24,7 +24,7 @@ Status is Active or Inactive -Uses Boto, read here for the list of ways to configure your AWS credentials: +Uses the Boto library, read here for the list of ways to configure your AWS credentials: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html From 7c29886514bf91d4faa89b972566f3afa59b721d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 12:08:54 +0000 Subject: [PATCH 0527/2779] added aws_users_pw_last_used.py --- aws_users_pw_last_used.py | 114 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100755 aws_users_pw_last_used.py diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py new file mode 100755 index 000000000..645121510 --- /dev/null +++ b/aws_users_pw_last_used.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-19 11:43:25 +0000 (Thu, 19 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Lists all AWS IAM users keys along with the dates and days since their passwords were last used, optionally filtering +for users whose passwords haven't been used in > N days + +Output format is: + + + +Uses Boto, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html + +This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies + +See also the DevOps Bash Tools repo and The Advanced Nagios Plugins Collection for similar tools + +https://github.com/harisekhon/devops-bash-tools + +https://github.com/harisekhon/nagios-plugins + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import datetime +import os +import sys +from math import floor +import boto3 +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_float + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +class AWSUsersPasswordLastUsed(CLI): + + def __init__(self): + super(AWSUsersPasswordLastUsed, self).__init__() + self.age = None + self.now = None + self.timeout_default = 300 + + def add_options(self): + self.add_opt('-a', '--age', help='Return users with passwords last used more than N days ago') + + def process_args(self): + self.age = self.get_opt('age') + if self.age: + validate_float(self.age, 'age') + self.age = float(self.age) + + def run(self): + iam = boto3.client('iam') + user_paginator = iam.get_paginator('list_users') + self.now = datetime.datetime.utcnow() + for users_response in user_paginator.paginate(): + for user_item in users_response['Users']: + log.debug(user_item) + self.process_password_age(user_item) + log.info('Completed') + + def process_password_age(self, user_item): + # already cast to datetime.datetime with tzinfo + user = user_item['UserName'] + if 'PasswordLastUsed' in user_item: + password_last_used = user_item['PasswordLastUsed'] + # removing tzinfo for comparison to avoid below error + # - both are UTC and this doesn't make much difference anyway + # TypeError: can't subtract offset-naive and offset-aware datetimes + datetime_delta = self.now - password_last_used.replace(tzinfo=None) + days = int(floor(datetime_delta.total_seconds() / 86400)) + if self.age and days <= self.age: + return + else: + password_last_used = 'N/A' + days = 'N/A' + print('{user:20s}\t{password_last_used}\t({days:>3} days)'.format( + user=user, + password_last_used=password_last_used, + days=days)) + + +if __name__ == '__main__': + AWSUsersPasswordLastUsed().main() From 761e86ddffa7f64515899925a1e4d8fd1f892f66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 12:10:20 +0000 Subject: [PATCH 0528/2779] updated aws_users_pw_last_used.py --- aws_users_pw_last_used.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py index 645121510..ae3557778 100755 --- a/aws_users_pw_last_used.py +++ b/aws_users_pw_last_used.py @@ -104,9 +104,9 @@ def process_password_age(self, user_item): else: password_last_used = 'N/A' days = 'N/A' - print('{user:20s}\t{password_last_used}\t({days:>3} days)'.format( + print('{user:20s}\t{password_last_used:25s}\t({days:>3} days)'.format( user=user, - password_last_used=password_last_used, + password_last_used=str(password_last_used), # without str() format string breaks with :25 days=days)) From 95c7e70ed16b9a82ecdd6cbf83fdb0e8436ea8ed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:01:26 +0000 Subject: [PATCH 0529/2779] added json_to_yaml.py --- json_to_yaml.py | 120 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100755 json_to_yaml.py diff --git a/json_to_yaml.py b/json_to_yaml.py new file mode 100755 index 000000000..32f36e22e --- /dev/null +++ b/json_to_yaml.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-19 17:54:21 +0000 (Thu, 19 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +""" + +Tool to convert JSON to YAML + +Reads any given files as JSON and prints the equivalent YAML to stdout for piping or redirecting to a file. + +Directories if given are detected and recursed, processing all files in the directory tree ending in a .json suffix. + +Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from +standard input. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import json +import os +import re +import sys +import yaml +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import die, ERRORS, log, log_option + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class JsonToYaml(CLI): + + def __init__(self): + # Python 2.x + super(JsonToYaml, self).__init__() + # Python 3.x + # super().__init__() + self.re_json_suffix = re.compile(r'.*\.json$', re.I) + + @staticmethod + def json_to_yaml(content, filepath=None): + try: + _ = json.loads(content) + except (KeyError, ValueError) as _: + file_detail = '' + if filepath is not None: + file_detail = ' in file \'{0}\''.format(filepath) + die("Failed to parse JSON{0}: {1}".format(file_detail, _)) + return yaml.safe_dump(_) + + def run(self): + if not self.args: + self.args.append('-') + for arg in self.args: + if arg == '-': + continue + if not os.path.exists(arg): + print("'%s' not found" % arg) + sys.exit(ERRORS['WARNING']) + if os.path.isfile(arg): + log_option('file', arg) + elif os.path.isdir(arg): + log_option('directory', arg) + else: + die("path '%s' could not be determined as either a file or directory" % arg) + for arg in self.args: + self.process_path(arg) + + def process_path(self, path): + if path == '-' or os.path.isfile(path): + self.process_file(path) + elif os.path.isdir(path): + for root, _, files in os.walk(path): + for filename in files: + filepath = os.path.join(root, filename) + if self.re_json_suffix.match(filepath): + self.process_file(filepath) + else: + die("failed to determine if path '%s' is a file or directory" % path) + + def process_file(self, filepath): + log.debug('processing filepath \'%s\'', filepath) + if filepath == '-': + filepath = '' + if filepath == '': + self.json_to_yaml(sys.stdin.read()) + else: + with open(filepath) as _: + content = _.read() + print('---') + print(self.json_to_yaml(content, filepath=filepath)) + + +if __name__ == '__main__': + JsonToYaml().main() From 97396a6f9c998a254cf7961dc893ca299d488e44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:03:57 +0000 Subject: [PATCH 0530/2779] updated json_to_yaml.py --- json_to_yaml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/json_to_yaml.py b/json_to_yaml.py index 32f36e22e..078b2fbac 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -25,6 +25,10 @@ Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from standard input. +See also: + + https://www.commandlinefu.com/commands/view/12221/convert-json-to-yaml + """ from __future__ import absolute_import From b52aaa7aec8e830fa172f9915c9226fb87b31551 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:10:07 +0000 Subject: [PATCH 0531/2779] added tests/test_json_to_yaml.sh --- tests/test_json_to_yaml.sh | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 tests/test_json_to_yaml.sh diff --git a/tests/test_json_to_yaml.sh b/tests/test_json_to_yaml.sh new file mode 100755 index 000000000..4e4d2cae2 --- /dev/null +++ b/tests/test_json_to_yaml.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-19 18:04:15 +0000 (Thu, 19 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "$0")" && pwd)" + +cd "$srcdir"; + +. utils.sh +. ../bash-tools/lib/utils.sh + +section "JSON => YAML" + +cd .. + +tmpfile="$(mktemp json_to_yaml_test.XXXXX.yml)" +#echo "tmpfile is $tmpfile" + +trap "rm -f $tmpfile" $TRAP_SIGNALS + +for x in ./cloudformation/centos7-12nodes-encrypted.json tests/data/embedded_double_quotes.json; do + echo "running json_to_yaml.py $x" + ./json_to_yaml.py "$x" > "$tmpfile" + echo "now validating generated yaml" + ./validate_yaml.py "$tmpfile" + echo +done + +echo "recursing directory to convert all json to yaml > /dev/null" +./json_to_yaml.py cloudformation/ >/dev/null +echo "Success" From dbd24050ddfee24c4863fd4ad856d275c2f4a898 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:10:21 +0000 Subject: [PATCH 0532/2779] updated test_json_to_xml.sh --- tests/test_json_to_xml.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 81a8f2669..095ec85f1 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -29,10 +29,10 @@ tmpfile="$(mktemp json_to_xml_test.XXXXX.xml)" trap "rm -f $tmpfile" $TRAP_SIGNALS -echo "running json_to_xml.py": +echo "running json_to_xml.py:" ../json_to_xml.py data/test.json | tee "$tmpfile" echo -echo "now validating generated xml": +echo "now validating generated xml:" ../validate_xml.py "$tmpfile" echo From e5840fbe4a8c21cfa4c57f09f57d2c02c4a25707 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:12:36 +0000 Subject: [PATCH 0533/2779] updated test_json_to_yaml.sh --- tests/test_json_to_yaml.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_json_to_yaml.sh b/tests/test_json_to_yaml.sh index 4e4d2cae2..47c39af6e 100755 --- a/tests/test_json_to_yaml.sh +++ b/tests/test_json_to_yaml.sh @@ -39,6 +39,9 @@ for x in ./cloudformation/centos7-12nodes-encrypted.json tests/data/embedded_dou echo done -echo "recursing directory to convert all json to yaml > /dev/null" -./json_to_yaml.py cloudformation/ >/dev/null +echo "recursing directory to convert all json files under a directory tree to yaml" +./json_to_yaml.py cloudformation/ > "$tmpfile" +# TODO: fix validate_yaml.py to work on multi-yamls with --- and re-enable +#echo "now validating generated yaml" +#./validate_yaml.py "$tmpfile" echo "Success" From 4bdcbd0027d71811c13a909ede0cad66a60974c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Dec 2019 18:40:25 +0000 Subject: [PATCH 0534/2779] updated json_to_yaml.py --- json_to_yaml.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/json_to_yaml.py b/json_to_yaml.py index 078b2fbac..231b6ae1c 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -25,6 +25,8 @@ Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from standard input. +Written to convert old AWS CloudFormation json templates to yaml + See also: https://www.commandlinefu.com/commands/view/12221/convert-json-to-yaml From 375ca823d386750ce6a74e536853fac666c5c27b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:41:32 +0000 Subject: [PATCH 0535/2779] fixed printing from stdin --- json_to_yaml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json_to_yaml.py b/json_to_yaml.py index 231b6ae1c..b3c2b4501 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -56,7 +56,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2.0' class JsonToYaml(CLI): @@ -114,7 +114,7 @@ def process_file(self, filepath): if filepath == '-': filepath = '' if filepath == '': - self.json_to_yaml(sys.stdin.read()) + print(self.json_to_yaml(sys.stdin.read())) else: with open(filepath) as _: content = _.read() From c721b83b7b45431d5f142988861bc8c11d37ba9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:44:24 +0000 Subject: [PATCH 0536/2779] updated test_json_to_xml.sh --- tests/test_json_to_xml.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 095ec85f1..956ed23ee 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -24,15 +24,15 @@ cd "$srcdir"; section "JSON => XML" -tmpfile="$(mktemp json_to_xml_test.XXXXX.xml)" +#tmpfile="$(mktemp json_to_xml_test.XXXXX.xml)" #echo "tmpfile is $tmpfile" -trap "rm -f $tmpfile" $TRAP_SIGNALS +#trap "rm -f $tmpfile" $TRAP_SIGNALS echo "running json_to_xml.py:" -../json_to_xml.py data/test.json | tee "$tmpfile" +../json_to_xml.py data/test.json | tee /dev/stderr | validate_xml.py echo -echo "now validating generated xml:" -../validate_xml.py "$tmpfile" +echo "running json_to_xml.py from stdin:" +../json_to_xml.py < data/test.json | tee /dev/stderr | validate_xml.py echo From faa993267ac7d159888a8e6d00a46bfa19493c80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:47:14 +0000 Subject: [PATCH 0537/2779] updated test_xml_to_json.sh --- tests/test_xml_to_json.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index 17d15b2c7..3728aa487 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -24,15 +24,14 @@ cd "$srcdir"; section "XML => JSON" +cd .. + for x in simple.xml plant_catalog.xml; do - tmpfile="$(mktemp xml_to_json_test.XXXXX.xml)" - trap "rm -f $tmpfile" $TRAP_SIGNALS + x="tests/data/$x" + #tmpfile="$(mktemp xml_to_json_test.XXXXX.xml)" + #trap "rm -f $tmpfile" $TRAP_SIGNALS echo "running xml_to_json.py on $x": - ../xml_to_json.py "data/$x" > "$tmpfile" - echo - echo "now validating generated json": - ../validate_json.py "$tmpfile" + ./xml_to_json.py "$x" | tee /dev/stderr | ./validate_json.py echo - rm -f "$tmpfile" echo "=========" done From df50404af140b8dfbfc144fd8364ef754f29d90f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:55:21 +0000 Subject: [PATCH 0538/2779] removed duplicate STDIN print --- validate_json.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/validate_json.py b/validate_json.py index aab399e41..03cc0aa7a 100755 --- a/validate_json.py +++ b/validate_json.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.11.0' +__version__ = '0.11.1' class JsonValidatorTool(CLI): @@ -297,8 +297,6 @@ def check(self, filename): if filename == '-': filename = '' self.filename = filename - self.valid_json_msg = '{0} => JSON OK'.format(filename) - self.invalid_json_msg = '{0} => JSON INVALID'.format(filename) single_quotes = '(found single quotes not double quotes)' self.valid_json_msg_single_quotes = '{0} {1}'.format(self.valid_json_msg, single_quotes) self.invalid_json_msg_single_quotes = '{0} {1}'.format(self.invalid_json_msg, single_quotes) From a541e6523ed9936673b24d8f9b450eafc2718cbf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:57:58 +0000 Subject: [PATCH 0539/2779] updated test_xml_to_json.sh --- tests/test_xml_to_json.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index 3728aa487..5a1b6fffc 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -26,12 +26,18 @@ section "XML => JSON" cd .. -for x in simple.xml plant_catalog.xml; do - x="tests/data/$x" - #tmpfile="$(mktemp xml_to_json_test.XXXXX.xml)" - #trap "rm -f $tmpfile" $TRAP_SIGNALS - echo "running xml_to_json.py on $x": - ./xml_to_json.py "$x" | tee /dev/stderr | ./validate_json.py - echo - echo "=========" -done +testdata="tests/data/simple.xml" + +echo "running xml_to_json.py on $testdata": +./xml_to_json.py "$testdata" | tee /dev/stderr | ./validate_json.py +echo + +echo "running xml_to_json.py on stdin < $testdata": +./xml_to_json.py < "$testdata" | tee /dev/stderr | ./validate_json.py +echo + +echo "running xml_to_json.py on tests/data/plant_catalog.xml": +./xml_to_json.py "tests/data/plant_catalog.xml" | ./validate_json.py +echo +echo "XML to JSON tests succeeded!" +echo From 9b7ebae047aa3482616c52a44c27109ead869f7a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:58:24 +0000 Subject: [PATCH 0540/2779] updated test_json_to_xml.sh --- tests/test_json_to_xml.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 956ed23ee..320cb86e8 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -36,3 +36,6 @@ echo echo "running json_to_xml.py from stdin:" ../json_to_xml.py < data/test.json | tee /dev/stderr | validate_xml.py echo + +echo "JSON to XML tests succeeded!" +echo From f49bc2f9332b3cb9349a012d14d1203aeebe444a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:58:43 +0000 Subject: [PATCH 0541/2779] fixed printing from stdin input --- json_to_xml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json_to_xml.py b/json_to_xml.py index ca2afcafb..03f5b27fa 100755 --- a/json_to_xml.py +++ b/json_to_xml.py @@ -52,7 +52,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2.0' class JsonToXml(CLI): @@ -110,7 +110,7 @@ def process_file(self, filepath): if filepath == '-': filepath = '' if filepath == '': - self.json_to_xml(sys.stdin.read()) + print(self.json_to_xml(sys.stdin.read())) else: with open(filepath) as _: content = _.read() From a0ef0b9b60d992755ebf824e57df8b477c84ee99 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 10:58:52 +0000 Subject: [PATCH 0542/2779] fixed printing from stdin input --- xml_to_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xml_to_json.py b/xml_to_json.py index 10484bfce..a82ea8adf 100755 --- a/xml_to_json.py +++ b/xml_to_json.py @@ -51,7 +51,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2.0' class XmlToJson(CLI): @@ -116,7 +116,7 @@ def process_file(self, filepath): if filepath == '-': filepath = '' if filepath == '': - self.xml_to_json(sys.stdin.read()) + print(self.xml_to_json(sys.stdin.read())) else: with open(filepath) as _: content = _.read() From 25756d0de5ecc23b7a23d577f89434469900c448 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 12:15:24 +0000 Subject: [PATCH 0543/2779] added tests/test_xml_to_yaml.sh --- tests/test_xml_to_yaml.sh | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 tests/test_xml_to_yaml.sh diff --git a/tests/test_xml_to_yaml.sh b/tests/test_xml_to_yaml.sh new file mode 100755 index 000000000..6f93a6ddf --- /dev/null +++ b/tests/test_xml_to_yaml.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "$0")" && pwd)" + +cd "$srcdir"; + +# shellcheck disable=SC1091 +. utils.sh + +# shellcheck disable=SC1091 +. ../bash-tools/lib/utils.sh + +section "XML => YAML" + +cd .. + +testdata="tests/data/simple.xml" + +echo "running xml_to_yaml.py on $testdata": +./xml_to_yaml.py "$testdata" | tee /dev/stderr | ./validate_yaml.py +echo + +echo "running xml_to_yaml.py on stdin < $testdata": +./xml_to_yaml.py < "$testdata" | tee /dev/stderr | ./validate_yaml.py +echo + +echo "running xml_to_yaml.py on tests/data/plant_catalog.xml": +./xml_to_yaml.py "tests/data/plant_catalog.xml" | ./validate_yaml.py +echo +echo "XML to yaml tests succeeded!" +echo From 49afa470df18b72502864bc0fd59aab7df74fdbb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 12:15:35 +0000 Subject: [PATCH 0544/2779] updated test_xml_to_json.sh --- tests/test_xml_to_json.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index 5a1b6fffc..8d01e3750 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -19,7 +19,10 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir"; +# shellcheck disable=SC1091 . utils.sh + +# shellcheck disable=SC1091 . ../bash-tools/lib/utils.sh section "XML => JSON" From f4b327709c1efdd570a89beb1c10789d23f388e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 12:16:52 +0000 Subject: [PATCH 0545/2779] updated test_json_to_yaml.sh --- tests/test_json_to_yaml.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_json_to_yaml.sh b/tests/test_json_to_yaml.sh index 47c39af6e..fc2b58868 100755 --- a/tests/test_json_to_yaml.sh +++ b/tests/test_json_to_yaml.sh @@ -19,7 +19,10 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir"; +# shellcheck disable=SC1091 . utils.sh + +# shellcheck disable=SC1091 . ../bash-tools/lib/utils.sh section "JSON => YAML" @@ -29,7 +32,9 @@ cd .. tmpfile="$(mktemp json_to_yaml_test.XXXXX.yml)" #echo "tmpfile is $tmpfile" -trap "rm -f $tmpfile" $TRAP_SIGNALS +# want var splitting +# shellcheck disable=SC2086 +trap 'rm -f "$tmpfile"' $TRAP_SIGNALS for x in ./cloudformation/centos7-12nodes-encrypted.json tests/data/embedded_double_quotes.json; do echo "running json_to_yaml.py $x" From 94e0e8848ca944b8103ae6c618d220b1dfeb4f3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 12:17:11 +0000 Subject: [PATCH 0546/2779] added xml_to_yaml.py --- xml_to_yaml.py | 131 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100755 xml_to_yaml.py diff --git a/xml_to_yaml.py b/xml_to_yaml.py new file mode 100755 index 000000000..6643a970a --- /dev/null +++ b/xml_to_yaml.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-19 18:19:34 +0000 (Thu, 19 Dec 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +""" + +Tool to convert XML to YAML + +Reads any given files as XML and prints the equivalent YAML to stdout for piping or redirecting to a file. + +Directories if given are detected and recursed, processing all files in the directory tree ending in a .xml suffix. + +Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from +standard input. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import json +import os +import re +import sys +import xml +import xmltodict +import yaml +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import die, ERRORS, log, log_option + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class XmlToYaml(CLI): + + def __init__(self): + # Python 2.x + super(XmlToYaml, self).__init__() + # Python 3.x + # super().__init__() + self.indent = None + self.re_xml_suffix = re.compile(r'.*\.xml$', re.I) + + def add_options(self): + self.add_opt('-p', '--pretty', action='store_true', help='Pretty Print the resulting YAML') + + @staticmethod + def xml_to_yaml(content, filepath=None): + try: + _ = xmltodict.parse(content) + except xml.parsers.expat.ExpatError as _: + file_detail = '' + if filepath is not None: + file_detail = ' in file \'{0}\''.format(filepath) + die("Failed to parse XML{0}: {1}".format(file_detail, _)) + # xmltodict returns a unicode OrderedDict so need to make it a plain dict to come out properly not like: + # !!python/object/apply:collections.OrderedDict + yaml_string = yaml.safe_dump(json.loads(json.dumps(_)), encoding='utf-8', sort_keys=True) + return yaml_string + + def run(self): + if self.get_opt('pretty'): + log_option('pretty', True) + self.indent = 4 + if not self.args: + self.args.append('-') + for arg in self.args: + if arg == '-': + continue + if not os.path.exists(arg): + print("'{}' not found".format(arg)) + sys.exit(ERRORS['WARNING']) + if os.path.isfile(arg): + log_option('file', arg) + elif os.path.isdir(arg): + log_option('directory', arg) + else: + die("path '{}' could not be determined as either a file or directory".format(arg)) + for arg in self.args: + self.process_path(arg) + + def process_path(self, path): + if path == '-' or os.path.isfile(path): + self.process_file(path) + elif os.path.isdir(path): + for root, _, files in os.walk(path): + for filename in files: + filepath = os.path.join(root, filename) + if self.re_xml_suffix.match(filepath): + self.process_file(filepath) + else: + die("failed to determine if path '{}' is a file or directory".format(path)) + + def process_file(self, filepath): + log.debug("processing filepath '%s'", filepath) + if filepath == '-': + filepath = '' + if filepath == '': + print(self.xml_to_yaml(sys.stdin.read())) + else: + with open(filepath) as _: + content = _.read() + print(self.xml_to_yaml(content, filepath=filepath)) + + +if __name__ == '__main__': + XmlToYaml().main() From d6c6972ee676e6af0976913f92dc2c65a8e65961 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 15:45:13 +0000 Subject: [PATCH 0547/2779] added days field to output --- aws_users_access_key_age.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 89dbbfa02..5c62e2333 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -50,6 +50,7 @@ import datetime import os import sys +from math import ceil import boto3 libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) @@ -64,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' class AWSUsersAccessKeysAge(CLI): @@ -85,7 +86,6 @@ def process_args(self): if self.age: validate_float(self.age, 'age') self.age = float(self.age) - self.age = self.age * 86400 def run(self): iam = boto3.client('iam') @@ -107,18 +107,21 @@ def process_key(self, keys_response, username): if self.only_active_keys and status != 'Active': continue create_date = access_key_item['CreateDate'] + # already cast to datetime.datetime with tzinfo + #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') + # removing tzinfo for comparison to avoid below error + # - both areOA UTC and this doesn't make much difference anyway + # TypeError: can't subtract offset-naive and offset-aware datetimes + age_timedelta = self.now - create_date.replace(tzinfo=None) + age_days = int(ceil(age_timedelta.total_seconds() / 86400.0)) if self.age: - # already cast to datetime.datetime with tzinfo - #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') - # removing tzinfo for comparison to avoid below error - # - both are UTC and this doesn't make much difference anyway - # TypeError: can't subtract offset-naive and offset-aware datetimes - if (self.now - create_date.replace(tzinfo=None)).total_seconds() < self.age: + if age_days < self.age: continue - print('{user:20}\t{status:8}\t{date}'.format( + print('{user:20}\t{status:8}\t{date}\t({days:>3} days)'.format( user=username, status=status, - date=create_date)) + date=create_date, + days=age_days)) if __name__ == '__main__': From b7de1020d80ea318ab93affbbdf6b8c6824f52a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 15:53:17 +0000 Subject: [PATCH 0548/2779] updated aws_users_pw_last_used.py --- aws_users_pw_last_used.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py index ae3557778..b2429b8d6 100755 --- a/aws_users_pw_last_used.py +++ b/aws_users_pw_last_used.py @@ -86,10 +86,10 @@ def run(self): for users_response in user_paginator.paginate(): for user_item in users_response['Users']: log.debug(user_item) - self.process_password_age(user_item) + self.process_password_last_used(user_item) log.info('Completed') - def process_password_age(self, user_item): + def process_password_last_used(self, user_item): # already cast to datetime.datetime with tzinfo user = user_item['UserName'] if 'PasswordLastUsed' in user_item: From 8f4944a768c28fe7b61299e7c2ed0f02e342e1cf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 18:09:37 +0000 Subject: [PATCH 0549/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 5c62e2333..9d39caf33 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -108,7 +108,7 @@ def process_key(self, keys_response, username): continue create_date = access_key_item['CreateDate'] # already cast to datetime.datetime with tzinfo - #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%d %H:%M:%S%z') + #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%dT%H:%M:%S%z') # removing tzinfo for comparison to avoid below error # - both areOA UTC and this doesn't make much difference anyway # TypeError: can't subtract offset-naive and offset-aware datetimes From 05f0e995e64d2e8afd31fcc0f298736e6aa86d00 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Dec 2019 18:15:41 +0000 Subject: [PATCH 0550/2779] added aws_users_last_used.py --- aws_users_last_used.py | 151 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 aws_users_last_used.py diff --git a/aws_users_last_used.py b/aws_users_last_used.py new file mode 100755 index 000000000..9e7d47e83 --- /dev/null +++ b/aws_users_last_used.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019) +# +# https://github.com/harisekhon/nagios-plugins +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Find AWS IAM user accounts last used age in days using the most recently used among their timestamp and access keys + +Optionally filters to only users > N days old to find old user accounts + +Generates an IAM credential report, then parses it to determine the time since each user's password +and access keys were last used + +Output: + + + +Uses the Boto python library, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html + +See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools + +- https://github.com/harisekhon/devops-bash-tools +- https://github.com/harisekhon/nagios-plugins + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import csv +import os +import sys +import time +import traceback +from datetime import datetime +from io import StringIO +from math import floor +import boto3 +from botocore.exceptions import ClientError +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_int + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class AWSUsersLastUsed(CLI): + + def __init__(self): + # Python 2.x + super(AWSUsersLastUsed, self).__init__() + # Python 3.x + # super().__init__() + self.age = None + self.now = None + self.msg = 'AWSUsersLastUsed msg not defined' + + def add_options(self): + self.add_opt('-a', '--age', type=float, + help='Filters to show only accounts last used more than N days ago') + + def process_args(self): + self.no_args() + self.age = self.get_opt('age') + if self.age is not None: + validate_int(self.age, 'age') + + def run(self): + iam = boto3.client('iam') + log.info('generating credentials report') + while True: + result = iam.generate_credential_report() + log.debug('%s', result) + if result['State'] == 'COMPLETE': + log.info('credentials report generated') + break + log.info('waiting for credentials report') + time.sleep(1) + try: + result = iam.get_credential_report() + except ClientError as _: + raise + csv_content = result['Content'] + log.debug('%s', csv_content) + filehandle = StringIO(unicode(csv_content)) + filehandle.seek(0) + csvreader = csv.reader(filehandle) + headers = csvreader.next() + assert headers[0] == 'user' + assert headers[4] == 'password_last_used' + assert headers[10] == 'access_key_1_last_used_date' + assert headers[15] == 'access_key_2_last_used_date' + self.now = datetime.utcnow() + for row in csvreader: + self.process_user(row) + + def process_user(self, row): + log.debug('processing user: %s', row) + user = row[0] + password_last_used = row[4] + access_key_1_last_used_date = row[10] + access_key_2_last_used_date = row[15] + log.debug('user: %s, password_last_used: %s, access_key_1_last_used_date: %s, access_key_2_last_used_date: %s', + user, password_last_used, access_key_1_last_used_date, access_key_2_last_used_date) + min_age = None + for _ in [password_last_used, access_key_1_last_used_date, access_key_2_last_used_date]: + if _ == 'N/A': + continue + # %z not working in Python 2.7 but we already know it's +00:00 + _datetime = datetime.strptime(_.split('+')[0], '%Y-%m-%dT%H:%M:%S') + age_timedelta = self.now - _datetime.replace(tzinfo=None) + age_days = int(floor(age_timedelta.total_seconds() / 86400.0)) + if min_age is None or age_days < min_age: + min_age = age_days + if self.age and min_age <= self.age: + return + print('{user:20}\t{days:>3}\t{password_last_used:25}\t'\ + .format(user=user, + days=min_age, + password_last_used=password_last_used) + + '{access_key_1_last_used_date:25}\t{access_key_2_last_used_date:25}'\ + .format(access_key_1_last_used_date=access_key_1_last_used_date, + access_key_2_last_used_date=access_key_2_last_used_date)) + + +if __name__ == '__main__': + AWSUsersLastUsed().main() From 81edcbdca1271cfb8b153f2284de551003b36470 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 15:30:10 +0000 Subject: [PATCH 0551/2779] imported hive_tables_row_counts.py from a while ago --- hive_tables_row_counts.py | 201 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100755 hive_tables_row_counts.py diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py new file mode 100755 index 000000000..2f834a401 --- /dev/null +++ b/hive_tables_row_counts.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to a HiveServer2 and get rows counts for all tables in all databases, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Hive 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +import impala +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' +] + +port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Gets row counts for all Hive tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='HiveServer2 host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), + help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, 'default') + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + try: + get_row_counts(conn, args, database, table, partition_regex) + except impala.error.OperationalError as _: + log.error(_) + +def get_row_counts(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {db}'.format(db=database)) + try: + partition_cursor.execute('show partitions {table}'.format(table=table)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + for result in partition_cursor: + row_count = result[0] + print('{db}.{table}.{key}={value}\t{row_count}'.format(\ + db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) + except impala.error.OperationalError as _: + if 'is not a partitioned table' not in str(_): + raise + log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", + database, table) + with conn.cursor() as table_cursor: + log.info("running SELECT COUNT(*) FROM %s.%s", database, table) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + + +if __name__ == '__main__': + main() From 6657ca03d48caa88f36fdefcc015f7bb7b1d0386 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 15:42:31 +0000 Subject: [PATCH 0552/2779] quick fork of hive_tables_row_counts.py --- impala_tables_row_counts.py | 199 ++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100755 impala_tables_row_counts.py diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py new file mode 100755 index 000000000..de4545a41 --- /dev/null +++ b/impala_tables_row_counts.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala node and get rows counts for all tables in all databases, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Impala 2.7.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.impalaserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +import impala +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'IMPALA_HOST', + 'HOST' +] + +port_envs = [ + 'IMPALA_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Gets row counts for all impala tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='Impala node ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), + help='Impala port (default: 21050, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='impala', + help='Service principal (default: \'impala\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, 'default') + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + try: + get_row_counts(conn, args, database, table, partition_regex) + except impala.error.OperationalError as _: + log.error(_) + +def get_row_counts(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {db}'.format(db=database)) + try: + partition_cursor.execute('show partitions {table}'.format(table=table)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + for result in partition_cursor: + row_count = result[0] + print('{db}.{table}.{key}={value}\t{row_count}'.format(\ + db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + if 'Table is not partitioned' not in str(_): + raise + log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", + database, table) + with conn.cursor() as table_cursor: + log.info("running SELECT COUNT(*) FROM %s.%s", database, table) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + + +if __name__ == '__main__': + main() From ed75ff7c9b989c0fddb81c45d979d1bd7ec7df8a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:02:41 +0000 Subject: [PATCH 0553/2779] added hive_foreach_table.py --- hive_foreach_table.py | 183 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100755 hive_foreach_table.py diff --git a/hive_foreach_table.py b/hive_foreach_table.py new file mode 100755 index 000000000..8fcb5b429 --- /dev/null +++ b/hive_foreach_table.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to connect to a HiveServer2 and execute a query for all tables in all databases, +or only those matching given db / table regexes + +Useful for getting row counts of all tables or analyzing tables: + +eg. + +hive_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' +hive_foreach_table.py --query 'ANALYZE TABLE {db}.{table} COMPUTE STATS' + +or just for today's partition: + +hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(DATE=$(date '+%Y-%m-%d')) COMPUTE STATS" + + +Tested on CDH 5.10, Hive 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +import impala +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' +] + +port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching Hive table") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='HiveServer2 host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), + help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ + ' (replaces {db} and {table} in the query string with each table and its database)') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, 'default') + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + try: + query = args.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = args.query.format(table=table) + try: + log.info("running %s", query) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute(query) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + log.error(_) + + +if __name__ == '__main__': + main() From 2573d1b1bb1958be507fd82c9d6060ed709c5ff7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:04:02 +0000 Subject: [PATCH 0554/2779] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index de4545a41..616e00e3f 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -27,7 +27,7 @@ If you get an error like this: -ERROR:impala.impalaserver2:Failed to open transport (tries_left=1) +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) ... TTransportException: TSocket read 0 bytes From de61d68af99584c5fabe386a54e1c6bc31acdf90 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:08:34 +0000 Subject: [PATCH 0555/2779] added impala_foreach_table.py --- impala_foreach_table.py | 181 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100755 impala_foreach_table.py diff --git a/impala_foreach_table.py b/impala_foreach_table.py new file mode 100755 index 000000000..e0bb42856 --- /dev/null +++ b/impala_foreach_table.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to connect to a Impala and execute a query for all tables in all databases, +or only those matching given db / table regexes + +Useful for getting row counts of all tables or analyzing tables: + +eg. + +impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' +impala_foreach_table.py --query 'COMPUTE STATS {table}' + +or just for today's partition: + +impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(DATE=$(date '+%Y-%m-%d'))" + + +Tested on CDH 5.10, Impala 2.7.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +import impala +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'IMPALA_HOST', + 'HOST' +] + +port_envs = [ + 'IMPALA_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching Impala table") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='Impala host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), + help='Impala port (default: 21050, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ + ' (replaces {db} and {table} in the query string with each table and its database)') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='impala', + help='Service principal (default: \'impala\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + if args.verbose > 1 or os.getenv('DEBUG'): + log.setLevel(logging.DEBUG) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, 'default') + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + try: + query = args.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = args.query.format(table=table) + try: + log.info("running %s", query) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute(query) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + log.error(_) + + +if __name__ == '__main__': + main() From fb59b03655126a176c369f0696b4806f0b0d577e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:10:52 +0000 Subject: [PATCH 0556/2779] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 760534d27..73aae4c85 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,8 @@ Environment variables are supported for convenience and also to hide credentials - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query of statement for each matching table + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterate all matching tables in all databases (by partition if available) and output TSV of table names and row counts - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From dcaee727df1846f539631a5fac96f560150457e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:12:00 +0000 Subject: [PATCH 0557/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73aae4c85..105e8ae0a 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Environment variables are supported for convenience and also to hide credentials - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query of statement for each matching table + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterate all matching tables in all databases (by partition if available) and output TSV of table names and row counts - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. From 4182da460f1bd41d4e53f22cad1e96feb2cf6ea5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:23:53 +0000 Subject: [PATCH 0558/2779] updated impala_foreach_table.py --- impala_foreach_table.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index e0bb42856..7fcdc1814 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -171,10 +171,14 @@ def main(): # doesn't support parameterized query quoting from dbapi spec table_cursor.execute(query) for result in table_cursor: - row_count = result[0] - print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: log.error(_) + #except impala.error.ProgrammingError as _: + # # COMPUTE STATS returns no results + # if not 'Trying to fetch results on an operation with no results' in _: + # raise if __name__ == '__main__': From 12c059751ca251c9c9a00ea854da287a9b91d542 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:24:33 +0000 Subject: [PATCH 0559/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 8fcb5b429..c05225619 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -173,10 +173,14 @@ def main(): # doesn't support parameterized query quoting from dbapi spec table_cursor.execute(query) for result in table_cursor: - row_count = result[0] - print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: log.error(_) + #except impala.error.ProgrammingError as _: + # # COMPUTE STATS returns no results + # if not 'Trying to fetch results on an operation with no results' in _: + # raise if __name__ == '__main__': From 5bed4a290cafa2e263d979519ee3240afdf022ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:27:06 +0000 Subject: [PATCH 0560/2779] updated impala_foreach_table.py --- impala_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 7fcdc1814..e966475c2 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -16,7 +16,7 @@ """ -Tool to connect to a Impala and execute a query for all tables in all databases, +Tool to connect to an Impala node and execute a query for all tables in all databases, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: From 39bf4033f8841372621c1260e970021cf012022d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:27:50 +0000 Subject: [PATCH 0561/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index c05225619..7255c5097 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -28,7 +28,7 @@ or just for today's partition: -hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(DATE=$(date '+%Y-%m-%d')) COMPUTE STATS" +hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS" Tested on CDH 5.10, Hive 1.1.0 with Kerberos From 4c4f98c68578bb343e7d44b979852f575b2f811e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:27:55 +0000 Subject: [PATCH 0562/2779] updated impala_foreach_table.py --- impala_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index e966475c2..b95b5dbbe 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -28,7 +28,7 @@ or just for today's partition: -impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(DATE=$(date '+%Y-%m-%d'))" +impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" Tested on CDH 5.10, Impala 2.7.0 with Kerberos From 6793ccb767b0944d0d538be60d68006932214ff2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 16:59:18 +0000 Subject: [PATCH 0563/2779] updated impala_foreach_table.py --- impala_foreach_table.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index b95b5dbbe..506bbcefb 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -153,8 +153,14 @@ def main(): with conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') + try: + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise for table_row in table_cursor: table = table_row[0] if not table_regex.search(table): @@ -168,17 +174,19 @@ def main(): query = args.query.format(table=table) try: log.info("running %s", query) - # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute(query) - for result in table_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) + with conn.cursor() as query_cursor: + # doesn't support parameterized query quoting from dbapi spec + query_cursor.execute(query) + for result in query_cursor: + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: log.error(_) - #except impala.error.ProgrammingError as _: - # # COMPUTE STATS returns no results - # if not 'Trying to fetch results on an operation with no results' in _: - # raise + except impala.error.ProgrammingError as _: + log.error(_) + # COMPUTE STATS returns no results + if 'Trying to fetch results on an operation with no results' not in str(_): + raise if __name__ == '__main__': From 5fa52c0313ca40ee8eb2a0a2c83a13a97d034d82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:03:04 +0000 Subject: [PATCH 0564/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 7255c5097..b15bb3bff 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -63,7 +63,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -153,10 +153,16 @@ def main(): #db_conn = connect_db(args, database) #with db_conn.cursor() as table_cursor: with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise for table_row in table_cursor: table = table_row[0] if not table_regex.search(table): @@ -170,18 +176,23 @@ def main(): query = args.query.format(table=table) try: log.info("running %s", query) - # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute(query) - for result in table_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) + with conn.cursor() as query_cursor: + # doesn't support parameterized query quoting from dbapi spec + query_cursor.execute(query) + for result in query_cursor: + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: log.error(_) - #except impala.error.ProgrammingError as _: - # # COMPUTE STATS returns no results - # if not 'Trying to fetch results on an operation with no results' in _: - # raise + except impala.error.ProgrammingError as _: + log.error(_) + # COMPUTE STATS returns no results + if 'Trying to fetch results on an operation with no results' not in str(_): + raise if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From b64496eb1d1e5b7de2abf54e6f8324b55d44b130 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:03:08 +0000 Subject: [PATCH 0565/2779] updated impala_foreach_table.py --- impala_foreach_table.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 506bbcefb..c868ba808 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -63,7 +63,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -151,9 +151,9 @@ def main(): #db_conn = connect_db(args, database) #with db_conn.cursor() as table_cursor: with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) table_cursor.execute('use {}'.format(database)) table_cursor.execute('show tables') except impala.error.HiveServer2Error as _: From 18b8c75f73372262c13721e38c5e24649d872c61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:04:58 +0000 Subject: [PATCH 0566/2779] updated impala_foreach_table.py --- impala_foreach_table.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index c868ba808..bb730f8aa 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -190,4 +190,7 @@ def main(): if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From 3ba56bcf1e30529446ff50ab8c3165c4b75db036 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:05:17 +0000 Subject: [PATCH 0567/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 3058e28dc..ed9d58973 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -199,4 +199,7 @@ def main(): if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From a4d996e1d89e74f353243cf64dd1218162d0a24a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:05:28 +0000 Subject: [PATCH 0568/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 2f834a401..f474e6b93 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -198,4 +198,7 @@ def get_row_counts(conn, args, database, table, partition_regex): if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From e66e7d485d38fea3972722ee86d5a2c99e070d40 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:05:49 +0000 Subject: [PATCH 0569/2779] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 616e00e3f..6dcc0a259 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -196,4 +196,7 @@ def get_row_counts(conn, args, database, table, partition_regex): if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + print("Control-C", file=sys.stderr) From 6f963803faedd4a085b6237ff92f7d7ef7dcf613 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:20:19 +0000 Subject: [PATCH 0570/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index f474e6b93..4093fd8bd 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -51,7 +51,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -142,10 +142,16 @@ def main(): #db_conn = connect_db(args, database) #with db_conn.cursor() as table_cursor: with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise for table_row in table_cursor: table = table_row[0] if not table_regex.search(table): From dce68d1406831b763437168fdff44cfa29662597 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:20:28 +0000 Subject: [PATCH 0571/2779] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 6dcc0a259..c9bb01d5f 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -51,7 +51,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -140,10 +140,16 @@ def main(): #db_conn = connect_db(args, database) #with db_conn.cursor() as table_cursor: with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise for table_row in table_cursor: table = table_row[0] if not table_regex.search(table): From c2de8223bacbe95e5d15348bcd64cfd6787d51a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:30:55 +0000 Subject: [PATCH 0572/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 54 +++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index ed9d58973..a0ff49f89 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -67,25 +67,11 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.2.1' +__version__ = '0.3.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) -host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'IMPALA_HOST', - 'HOST' -] - -port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'IMPALA_PORT', - 'PORT' -] - def getenvs(keys, default=None): for key in keys: value = os.getenv(key) @@ -94,17 +80,40 @@ def getenvs(keys, default=None): return default def parse_args(): + default_port = 10000 + default_service_name = 'hive' + host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' + ] + port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' + ] + + if 'impala' in sys.argv[0]: + default_port = 21050 + default_service_name = 'impala' + host_envs = [ + 'IMPALA_HOST', + 'HOST' + ] + port_envs = [ + 'IMPALA_PORT', + 'PORT' + ] parser = argparse.ArgumentParser( description="Dumps all Hive / Impala schemas, tables, columns and types to CSV format on stdout") parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ help='HiveServer2 / Impala host ' + \ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), - help='HiveServer2 / Impala port (default: 10000 if called as hive, ' + \ - '21050 if called as impala, $' + \ + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), + help='HiveServer2 / Impala port (default: {}, '.format(default_port) + \ ', $'.join(port_envs) + ')') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='hive', + parser.add_argument('-n', '--krb5-service-name', default=default_service_name, help='Service principal (default: \'hive\', or \'impala\' if called as impala_schemas_csv.py)') parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') # must set type to str otherwise csv module gives this error on Python 2.7: @@ -121,13 +130,6 @@ def parse_args(): if args.verbose > 1 or os.getenv('DEBUG'): log.setLevel(logging.DEBUG) - if 'impala' in sys.argv[0]: - if args.krb5_service_name == 'hive': - log.info('called as impala, setting service principal to impala') - args.krb5_service_name = 'impala' - if args.port == 10000: - log.info('called as impala, setting port to 21050') - args.port = 21050 return args def connect_db(args, database): From 9034a083146cb7d99eab558d2cea47b41e6374b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:38:49 +0000 Subject: [PATCH 0573/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index a0ff49f89..e780c5e69 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -114,7 +114,7 @@ def parse_args(): ', $'.join(port_envs) + ')') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') parser.add_argument('-n', '--krb5-service-name', default=default_service_name, - help='Service principal (default: \'hive\', or \'impala\' if called as impala_schemas_csv.py)') + help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') # must set type to str otherwise csv module gives this error on Python 2.7: # TypeError: "delimiter" must be string, not unicode From 969419bb81d169cce11136ff24583a6a8223903f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:41:27 +0000 Subject: [PATCH 0574/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index e780c5e69..f93bb9137 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -80,6 +80,7 @@ def getenvs(keys, default=None): return default def parse_args(): + name = 'HiveServer2' default_port = 10000 default_service_name = 'hive' host_envs = [ @@ -94,6 +95,7 @@ def parse_args(): ] if 'impala' in sys.argv[0]: + name = 'Impala' default_port = 21050 default_service_name = 'impala' host_envs = [ @@ -105,12 +107,12 @@ def parse_args(): 'PORT' ] parser = argparse.ArgumentParser( - description="Dumps all Hive / Impala schemas, tables, columns and types to CSV format on stdout") + description="Dumps all {} schemas, tables, columns and types to CSV format on stdout".format(name)) parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='HiveServer2 / Impala host ' + \ + help='{} host '.format(name) + \ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), - help='HiveServer2 / Impala port (default: {}, '.format(default_port) + \ + help='{} port (default: {}, '.format(name, default_port) + \ ', $'.join(port_envs) + ')') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') parser.add_argument('-n', '--krb5-service-name', default=default_service_name, From de6b88190d37839f43a85d3e57c2fba32ca74c41 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:43:08 +0000 Subject: [PATCH 0575/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 61 +++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 4093fd8bd..fef2194e2 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -16,10 +16,10 @@ """ -Connect to a HiveServer2 and get rows counts for all tables in all databases, +Connect to a HiveServer2 / Impala node and get rows counts for all tables in all databases, or only those matching given db / table / partition value regexes -Tested on CDH 5.10, Hive 1.1.0 with Kerberos +Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see @@ -56,18 +56,6 @@ logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) -host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' -] - -port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' -] - def getenvs(keys, default=None): for key in keys: value = os.getenv(key) @@ -76,19 +64,47 @@ def getenvs(keys, default=None): return default def parse_args(): + name = 'HiveServer2' + default_port = 10000 + default_service_name = 'hive' + host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' + ] + port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' + ] + + if 'impala' in sys.argv[0]: + name = 'Impala' + default_port = 21050 + default_service_name = 'impala' + host_envs = [ + 'IMPALA_HOST', + 'HOST' + ] + port_envs = [ + 'IMPALA_PORT', + 'PORT' + ] parser = argparse.ArgumentParser( - description="Gets row counts for all Hive tables/partitions matching database / table / partition regexes") + description="Gets row counts for all {} tables / partitions matching database / table / partition regexes"\ + .format(name)) parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='HiveServer2 host ' + \ + help='{} host '.format(name) + \ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), - help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), + help='{} port (default: {}, '.format(name, default_port) + \ + ', $'.join(port_envs) + ')') parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='hive', - help='Service principal (default: \'hive\')') + parser.add_argument('-n', '--krb5-service-name', default=default_service_name, + help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() @@ -190,7 +206,10 @@ def get_row_counts(conn, args, database, table, partition_regex): print('{db}.{table}.{key}={value}\t{row_count}'.format(\ db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) except impala.error.OperationalError as _: - if 'is not a partitioned table' not in str(_): + # Hive exception msg: is not a partitioned table + # Impala exception msg: Table is not partitioned + if 'is not a partitioned table' not in str(_) and \ + 'Table is not partitioned' not in str(_): raise log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", database, table) From 07549144f1e9b95e8dc2ac357e270cf18d65a56a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:43:16 +0000 Subject: [PATCH 0576/2779] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 209 +----------------------------------- 1 file changed, 1 insertion(+), 208 deletions(-) mode change 100755 => 120000 impala_tables_row_counts.py diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py deleted file mode 100755 index c9bb01d5f..000000000 --- a/impala_tables_row_counts.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python -# vim:ts=4:sts=4:sw=4:et -# -# Author: Hari Sekhon -# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) -# -# https://github.com/harisekhon/devops-python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn -# and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/harisekhon -# - -""" - -Connect to an Impala node and get rows counts for all tables in all databases, -or only those matching given db / table / partition value regexes - -Tested on CDH 5.10, Impala 2.7.0 with Kerberos - -Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see - -https://github.com/cloudera/impyla/issues/286 - -If you get an error like this: - -ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) -... -TTransportException: TSocket read 0 bytes - -then check your --kerberos and --ssl settings match the cluster's settings -(Thrift and Kerberos have the worst error messages ever) - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -import logging -import os -import re -import socket -import sys -import impala -from impala.dbapi import connect - -__author__ = 'Hari Sekhon' -__version__ = '0.2.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -host_envs = [ - 'IMPALA_HOST', - 'HOST' -] - -port_envs = [ - 'IMPALA_PORT', - 'PORT' -] - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - parser = argparse.ArgumentParser( - description="Gets row counts for all impala tables/partitions matching database / table / partition regexes") - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='Impala node ' + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), - help='Impala port (default: 21050, ' + ', $'.join(port_envs) + ')') - parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') - parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') - parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='impala', - help='Service principal (default: \'impala\')') - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - try: - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - partition_regex = re.compile(args.partition, re.I) - except re.error as _: - log.error('error in provided regex: %s', _) - sys.exit(3) - - conn = connect_db(args, 'default') - - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - if not database_regex.search(database): - log.debug("skipping database '%s', does not match regex '%s'", database, args.database) - continue - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, args.table) - continue - try: - get_row_counts(conn, args, database, table, partition_regex) - except impala.error.OperationalError as _: - log.error(_) - -def get_row_counts(conn, args, database, table, partition_regex): - log.info("getting partitions for database '%s' table '%s'", database, table) - with conn.cursor() as partition_cursor: - # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('use {db}'.format(db=database)) - try: - partition_cursor.execute('show partitions {table}'.format(table=table)) - for partitions_row in partition_cursor: - partition_key = partitions_row[0] - partition_value = partitions_row[1] - if not partition_regex.match(partition_value): - log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + - "value does not match regex '%s'", - database, - table, - partition_key, - partition_value, - args.partition) - continue - # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ - .format(db=database, table=table, key=partition_key, value=partition_value)) - for result in partition_cursor: - row_count = result[0] - print('{db}.{table}.{key}={value}\t{row_count}'.format(\ - db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) - except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - if 'Table is not partitioned' not in str(_): - raise - log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", - database, table) - with conn.cursor() as table_cursor: - log.info("running SELECT COUNT(*) FROM %s.%s", database, table) - # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) - for result in table_cursor: - row_count = result[0] - print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) - - -if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py new file mode 120000 index 000000000..65e53c4f2 --- /dev/null +++ b/impala_tables_row_counts.py @@ -0,0 +1 @@ +hive_tables_row_counts.py \ No newline at end of file From ee04d170bf2adc3f7f9261e6ecfaa8da755e6a9b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:46:12 +0000 Subject: [PATCH 0577/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index fef2194e2..80172c21a 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -51,7 +51,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -205,9 +205,9 @@ def get_row_counts(conn, args, database, table, partition_regex): row_count = result[0] print('{db}.{table}.{key}={value}\t{row_count}'.format(\ db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) - except impala.error.OperationalError as _: - # Hive exception msg: is not a partitioned table - # Impala exception msg: Table is not partitioned + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + # Hive impala.error.HiveServer2Error: is not a partitioned table + # Impala impala.error.HiveServer2Error: Table is not partitioned if 'is not a partitioned table' not in str(_) and \ 'Table is not partitioned' not in str(_): raise From 27fae644a72b331dfa6d5950c9b618f6a1f518d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:50:16 +0000 Subject: [PATCH 0578/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 60 ++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index b15bb3bff..e52eca238 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -16,7 +16,7 @@ """ -Tool to connect to a HiveServer2 and execute a query for all tables in all databases, +Tool to connect to a HiveServer2 / Impala node and execute a query for all tables in all databases, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: @@ -26,12 +26,17 @@ hive_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' hive_foreach_table.py --query 'ANALYZE TABLE {db}.{table} COMPUTE STATS' +impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' +impala_foreach_table.py --query 'COMPUTE STATS {table}' + or just for today's partition: hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS" +impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" + -Tested on CDH 5.10, Hive 1.1.0 with Kerberos +Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see @@ -68,18 +73,6 @@ logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) -host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' -] - -port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' -] - def getenvs(keys, default=None): for key in keys: value = os.getenv(key) @@ -88,19 +81,46 @@ def getenvs(keys, default=None): return default def parse_args(): - parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching Hive table") + name = 'HiveServer2' + default_port = 10000 + default_service_name = 'hive' + host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' + ] + port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' + ] + + if 'impala' in sys.argv[0]: + name = 'Impala' + default_port = 21050 + default_service_name = 'impala' + host_envs = [ + 'IMPALA_HOST', + 'HOST' + ] + port_envs = [ + 'IMPALA_PORT', + 'PORT' + ] + parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching {} table".format(name)) parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='HiveServer2 host ' + \ + help='{} host '.format(name) + \ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), - help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), + help='{} port (default: {}, '.format(name, default_port) + \ + ', $'.join(port_envs) + ')') parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ ' (replaces {db} and {table} in the query string with each table and its database)') parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='hive', - help='Service principal (default: \'hive\')') + parser.add_argument('-n', '--krb5-service-name', default=default_service_name, + help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() From 5fece61317c1c5dfd3486f74b38dce28d41615ba Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Dec 2019 17:50:27 +0000 Subject: [PATCH 0579/2779] updated impala_foreach_table.py --- impala_foreach_table.py | 197 +--------------------------------------- 1 file changed, 1 insertion(+), 196 deletions(-) mode change 100755 => 120000 impala_foreach_table.py diff --git a/impala_foreach_table.py b/impala_foreach_table.py deleted file mode 100755 index bb730f8aa..000000000 --- a/impala_foreach_table.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python -# vim:ts=4:sts=4:sw=4:et -# -# Author: Hari Sekhon -# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) -# -# https://github.com/harisekhon/devops-python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn -# and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/harisekhon -# - -""" - -Tool to connect to an Impala node and execute a query for all tables in all databases, -or only those matching given db / table regexes - -Useful for getting row counts of all tables or analyzing tables: - -eg. - -impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' -impala_foreach_table.py --query 'COMPUTE STATS {table}' - -or just for today's partition: - -impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" - - -Tested on CDH 5.10, Impala 2.7.0 with Kerberos - -Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see - -https://github.com/cloudera/impyla/issues/286 - -If you get an error like this: - -ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) -... -TTransportException: TSocket read 0 bytes - -then check your --kerberos and --ssl settings match the cluster's settings -(Thrift and Kerberos have the worst error messages ever) - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -import logging -import os -import re -import socket -import sys -import impala -from impala.dbapi import connect - -__author__ = 'Hari Sekhon' -__version__ = '0.2.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -host_envs = [ - 'IMPALA_HOST', - 'HOST' -] - -port_envs = [ - 'IMPALA_PORT', - 'PORT' -] - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching Impala table") - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='Impala host ' + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), - help='Impala port (default: 21050, ' + ', $'.join(port_envs) + ')') - parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ - ' (replaces {db} and {table} in the query string with each table and its database)') - parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') - parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default='impala', - help='Service principal (default: \'impala\')') - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - try: - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - except re.error as _: - log.error('error in provided regex: %s', _) - sys.exit(3) - - conn = connect_db(args, 'default') - - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - if not database_regex.search(database): - log.debug("skipping database '%s', does not match regex '%s'", database, args.database) - continue - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, args.table) - continue - try: - query = args.query.format(db=database, table=table) - except KeyError as _: - if _ == 'db': - query = args.query.format(table=table) - try: - log.info("running %s", query) - with conn.cursor() as query_cursor: - # doesn't support parameterized query quoting from dbapi spec - query_cursor.execute(query) - for result in query_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) - except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - log.error(_) - except impala.error.ProgrammingError as _: - log.error(_) - # COMPUTE STATS returns no results - if 'Trying to fetch results on an operation with no results' not in str(_): - raise - - -if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) diff --git a/impala_foreach_table.py b/impala_foreach_table.py new file mode 120000 index 000000000..44716e6dc --- /dev/null +++ b/impala_foreach_table.py @@ -0,0 +1 @@ +hive_foreach_table.py \ No newline at end of file From 28b4d4c0e46bf2ffc14d66d0c3c8c7f2e884970e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 11:21:01 +0000 Subject: [PATCH 0580/2779] increased timeout --- aws_users_last_used.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 9e7d47e83..eb2a1f963 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -77,6 +77,7 @@ def __init__(self): # super().__init__() self.age = None self.now = None + self.timeout_default = 300 self.msg = 'AWSUsersLastUsed msg not defined' def add_options(self): From 3bb4c74e84e2db5fa189a709f84d836866b15d79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 14:24:47 +0000 Subject: [PATCH 0581/2779] added AWS Access Key, Secret Key and STS token support, improved ARN matching, moved AWS matches to end as tokens are more generic --- anonymize.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/anonymize.py b/anonymize.py index aae935b51..15e54eb4c 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.7' +__version__ = '0.10.0' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -114,7 +114,6 @@ def __init__(self): self.hash_salt = None # order of iteration of application matters because we must do more specific matches before less specific ones self.anonymizations = OrderedDict([ - ('aws', False), ('ip_prefix', False), ('ip', False), ('subnet_mask', False), @@ -130,11 +129,12 @@ def __init__(self): ('screenos', False), ('junos', False), ('network', False), + ('windows', False), + ('aws', False), # access key, secret key, sts tokens etc are very generic so do them later ('fqdn', False), ('domain', False), ('hostname', False), #('proxy', False), - ('windows', False), ('custom', False), ]) self.exceptions = { @@ -287,10 +287,15 @@ def __init__(self): # arn:partition:service:region:account-id:resource-type/resource-id # arn:partition:service:region:account-id:resource-type:resource-id # eg. arn:aws:iam::123456789012:group/Development/product_1234/* - 'aws': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d+(:([^:/]+)[:/])[\w/-]+', + 'aws': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d+(:([^:/]+)[:/])[\w/.-]+', # arn:aws:s3:::my_corporate_bucket/Development/* #'aws2': r'\b(arn:aws:s3:::)[^/]+', - 'aws2': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d*:[\w/-]+', + 'aws2': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d*:[\w/.-]+', + # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html + 'aws3': r'\bAKIA[A-Za-z0-9]{16}\b', # access key + 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key + 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix + 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key # don't change hostname or fqdn regex without updating hash_hostnames() option parse # since that replaces these replacements and needs to match the grouping captures and surrounding format 'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex), @@ -398,6 +403,10 @@ def __init__(self): # arn:partition:service:region:account-id:resource-type:resource-id 'aws': r'\1\2<\3>', 'aws2': r'\1:', + 'aws3': r'', + 'aws4': r'', + 'aws5': r'', + 'aws6': r'', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', @@ -476,7 +485,7 @@ def add_options(self): self.add_opt('-a', '--all', action='store_true', help='Apply all anonymizations (careful this includes --host which can be overzealous and ' + \ 'match too many things, in which case try more targeted anonymizations below)') - self.add_opt('-w', '--aws', action='store_true', help='Apply AWS ARN anonymizations'), + self.add_opt('-w', '--aws', action='store_true', help='Apply AWS ARN anonymizations') self.add_opt('-C', '--custom', action='store_true', help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \ 'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \ From 05b9b5d7771c4485798c1ee38e998be40b81eb5f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 14:27:19 +0000 Subject: [PATCH 0582/2779] added aws access key, secret key and sts token tests --- tests/test_anonymize.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index fb0b8036a..53c641477 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -437,17 +437,32 @@ src[114]="arn:aws:iam::123456789012:group/hari" dest[114]="arn:aws:iam:::group/" src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*" -dest[115]="arn:aws:iam:::user/*" +dest[115]="arn:aws:iam:::user//*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" dest[116]="arn:aws:iam:::group/*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" -dest[116]="arn:aws:iam:::group/*" +dest[116]="arn:aws:iam:::group//*" src[117]="arn:aws:s3:::my_corporate_bucket/Development/*" dest[117]="arn:aws:s3:::*" +src[118]="AKIAIOSFODNN7EXAMPLE" +dest[118]="" + +src[119]="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +dest[119]="" + +src[120]="AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE" +dest[120]="" + +src[121]="ASIAIOSFODNN7EXAMPLE" +dest[121]="" + +src[122]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" +dest[122]="" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 997494885a75496d9da033fc4613f64cb78c47ce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 14:27:28 +0000 Subject: [PATCH 0583/2779] added .gitallowed --- .gitallowed | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .gitallowed diff --git a/.gitallowed b/.gitallowed new file mode 100644 index 000000000..bb1dbfb76 --- /dev/null +++ b/.gitallowed @@ -0,0 +1,9 @@ +AKIAIOSFODNN7EXAMPLE + +wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE + +ASIAIOSFODNN7EXAMPLE + +AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== From 00b28e50e5fafae21912f1ea311ad3cc41e6de47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:44:00 +0000 Subject: [PATCH 0584/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 105e8ae0a..64f5ac05e 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Environment variables are supported for convenience and also to hide credentials - hostnames / domains / FQDNs - email addresses - IP + MAC addresses - - AWS ARNs + - AWS Access Keys, Secret Keys, ARNs, STS tokens - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings From c0389f8f71f844e4d7b157164b506f4af9464fd5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:49:22 +0000 Subject: [PATCH 0585/2779] updated aws_users_pw_last_used.py --- aws_users_pw_last_used.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py index b2429b8d6..7e00ee4e0 100755 --- a/aws_users_pw_last_used.py +++ b/aws_users_pw_last_used.py @@ -16,7 +16,7 @@ """ -Lists all AWS IAM users keys along with the dates and days since their passwords were last used, optionally filtering +Lists all AWS IAM users dates since their passwords were last used, optionally filtering for users whose passwords haven't been used in > N days Output format is: From caf1a045309bcd92626587ebb9d20a8a5abc1bfa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:49:59 +0000 Subject: [PATCH 0586/2779] updated README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 64f5ac05e..84567c19b 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,10 @@ Environment variables are supported for convenience and also to hide credentials - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) +- [AWS](https://aws.amazon.com/): + - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days + - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove + - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Hadoop](http://hadoop.apache.org/) & NoSQL: - [Spark](https://spark.apache.org/) & Data Format Converters: - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter From dfe418cab5bc8a6ac9ec679b9370833bbd02ac39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:57:00 +0000 Subject: [PATCH 0587/2779] updated README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 84567c19b..f46a7cdfe 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,15 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) -### Hadoop, Spark / PySpark, HBase, Pig, Ambari, IPython and Linux Tools ### +### AWS, Docker, Spark / PySpark, Hadoop, HBase, Hive, Impala, Pig, Ambari, IPython and Linux Tools ### -A few of the Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options. +A few of the Cloud, Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options. For many more tools see the [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) and [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) repos which contains many Hadoop, NoSQL, Web and infrastructure tools and Nagios plugins. Hari Sekhon -Big Data Contractor, United Kingdom +Cloud Big Data Contractor, United Kingdom https://www.linkedin.com/in/harisekhon From 28aee028a1acd9ad43bfface188365316923e719 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 18:57:19 +0000 Subject: [PATCH 0588/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f46a7cdfe..99f03ce49 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ For many more tools see the [DevOps Perl Tools](https://github.com/harisekhon/pe Hari Sekhon -Cloud Big Data Contractor, United Kingdom +Cloud & Big Data Contractor, United Kingdom https://www.linkedin.com/in/harisekhon From 1ebfa94057253ba63a65d431d96fb3e6a7872610 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 19:00:53 +0000 Subject: [PATCH 0589/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99f03ce49..5e1ae26e2 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Hadoop, Docker, Kafka, Elasticsearch, RabbitMQ, Redis, HBase, Solr, Cassandra, ZooKeeper, HDFS, Yarn, Hive, Presto, Drill, Impala, Consul, Spark, Jenkins, Travis CI, Git, MySQL, Linux, DNS, Whois, SSL Certs, Yum Security Updates, Kubernetes, Mesos, Riak, MongoDB, Memcached, Couchbase, CouchDB, Neo4j, Ambari, Cloudera, Hortonworks, MapR etc. -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 80+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 100+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies * [HAProxy-configs](https://github.com/harisekhon/haproxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. From 5abdfebf9784cf7790f2a2f809e06b970f755da4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Dec 2019 19:02:28 +0000 Subject: [PATCH 0590/2779] Update README.md commented out spark elasticsearch repo --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5e1ae26e2..e316eba4e 100644 --- a/README.md +++ b/README.md @@ -348,7 +348,9 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library + You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: From 56a977bae7dbca69764d066f338bf6b5309ce066 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Dec 2019 18:26:54 +0000 Subject: [PATCH 0591/2779] updated aws_users_last_used.py --- aws_users_last_used.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index eb2a1f963..2657f03f7 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -23,6 +23,8 @@ Generates an IAM credential report, then parses it to determine the time since each user's password and access keys were last used +Requires iam:GenerateCredentialReport on resource: * + Output: From c4b18f924182c791483187f97aaf3e5df1e12899 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 15:18:24 +0000 Subject: [PATCH 0592/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c6a900361..270183668 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c6a9003610ddd2321dc28c2c7d69f98eb43ba7c7 +Subproject commit 270183668565e51f5c31fcf969c6cb57b07e0bfe From 389c4e603421747d2663228a3eed7a641cbbd48e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 15:18:47 +0000 Subject: [PATCH 0593/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index cf0435d86..f3fb9b2a9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit cf0435d8687fc0951c91ed5cb410a84bce31aca0 +Subproject commit f3fb9b2a9dedb8a4ed7e85183f1739ad2a63fd92 From fcd449247e0ca0957250b11df6e07062aed9ccd8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 15:20:51 +0000 Subject: [PATCH 0594/2779] switched to more forgiving safe_load_all() --- validate_yaml.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/validate_yaml.py b/validate_yaml.py index 285f75ef8..819943294 100755 --- a/validate_yaml.py +++ b/validate_yaml.py @@ -49,7 +49,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.2' +__version__ = '0.9.3' class YamlValidatorTool(CLI): @@ -86,7 +86,7 @@ def is_excluded(self, path): return False def check_yaml(self, content): - if isYaml(content): + if isYaml(content, safe_load_all=True): if self.get_opt('print'): print(content, end='') else: @@ -99,7 +99,6 @@ def check_yaml(self, content): if not self.get_opt('print'): if self.verbose > 2: try: - # TODO: doesn't validate network-policy.yaml in templates containing multiple yaml docs yaml.safe_load_all(content) except yaml.YAMLError as _: print(_) From 319ab52ca5560fad536f651bacf6406b4cd39a6b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 17:58:24 +0000 Subject: [PATCH 0595/2779] updated help.sh --- tests/help.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/help.sh b/tests/help.sh index 851451152..7b4edd786 100755 --- a/tests/help.sh +++ b/tests/help.sh @@ -22,7 +22,8 @@ cd "$srcdir/.."; # shellcheck disable=SC1091 . ./tests/utils.sh -for x in $(echo ./*.py 2>/dev/null); do +# shellcheck disable=SC2068 +for x in ${@:-$(echo ./*.py 2>/dev/null)}; do isExcluded "$x" && continue set +e echo "$x --help" @@ -33,7 +34,14 @@ for x in $(echo ./*.py 2>/dev/null); do if [ $status = 0 ]; then [[ "$x" =~ ambari_blueprints.py$ ]] && continue [[ "$x" =~ (hive|impala)_schemas_csv.py$ ]] && continue + [[ "$x" =~ (hive|impala)_foreach_table.py$ ]] && continue + [[ "$x" =~ (hive|impala)_tables_row_counts.py$ ]] && continue [[ "$x" =~ pythonpath.py$ ]] && continue + elif [ $status = 1 ]; then + if [[ "$x" =~ hdfs_find_replication_factor_1.py$ ]] && + ! python -c 'import krbV'; then # best effort, not available on Mac any more + continue + fi fi [ $status = 3 ] || { echo "status code for $x --help was $status not expected 3"; exit 1; } done From e45d8a1fd2b970855cf306d5218cf5a3d9d2fb61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 30 Dec 2019 18:14:08 +0000 Subject: [PATCH 0596/2779] updated test_dockerhub_search.sh --- tests/test_dockerhub_search.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index d625bf2db..73b8bf7b5 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -28,7 +28,9 @@ section "Testing DockerHub Show Tags" check './dockerhub_search.py centos' "DockerHub Search for CentOS" check './dockerhub_search.py harisekhon' "DockerHub Search for harisekhon" check './dockerhub_search.py harisekhon -n 30' "DockerHub Search for harisekhon -n 30" -check './dockerhub_search.py harisekhon/hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" +# this no longer works, API must have changed +#check './dockerhub_search.py harisekhon/hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" +check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" From a730b7a3956210946eb6d5a497eed24e6286c08a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Dec 2019 13:28:33 +0000 Subject: [PATCH 0597/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 270183668..384e5f8b3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 270183668565e51f5c31fcf969c6cb57b07e0bfe +Subproject commit 384e5f8b366fdeb6a73363dbf388824f5de5e7bc From a852ffa341d9b8a6b2945ee32f56f83bd48b9085 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Jan 2020 17:34:42 +0000 Subject: [PATCH 0598/2779] added hexanonymize.py --- hexanonymize.py | 116 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100755 hexanonymize.py diff --git a/hexanonymize.py b/hexanonymize.py new file mode 100755 index 000000000..600083615 --- /dev/null +++ b/hexanonymize.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-01-02 17:08:32 +0000 (Thu, 02 Jan 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +""" + +Tool to anonymize hex input but keeping the structure of the positions of numbers and digits + +Useful to retain the structure of ID formats + +Reads any given files or standard input and replaces each hex character with an incrementing number of letter (a-f) +printing to stdout for piping or redirecting to a file as per unix filter command standards + +Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from +standard input + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class HexAnonymize(CLI): + + def __init__(self): + # Python 2.x + super(HexAnonymize, self).__init__() + # Python 3.x + # super().__init__() + self.preserve_case = False + self.only_hex_alphas = False + + def add_options(self): + super(HexAnonymize, self).add_options() + self.add_opt('-c', '--case', action='store_true', help='Preserve case') + self.add_opt('-o', '--only-hex', action='store_true', + help='Only replace hex alpha chars (A-F, a-f), otherwise replaces all alphanumerics for safety') + + def process_options(self): + super(HexAnonymize, self).process_options() + self.preserve_case = self.get_opt('case') + self.only_hex_alphas = self.get_opt('only_hex') + + def hexanonymize(self, filehandle): + preserve_case = self.preserve_case + only_hex_alphas = self.only_hex_alphas + hex_alphas = ['a', 'b', 'c', 'd', 'e', 'f'] + for line in filehandle: + integer = 1 + letter = 'a' + for char in line: + if char.isdigit(): + char = integer + integer += 1 + if integer > 9: + integer = 0 + elif char.lower() in hex_alphas or (not only_hex_alphas and char.isalpha()): + if preserve_case and char.isupper(): + char = letter.upper() + else: + char = letter + letter = chr(ord(char) + 1) + if letter not in hex_alphas: + letter = 'a' + print(char, end='') + + + def run(self): + if not self.args: + self.args.append('-') + for arg in self.args: + if arg == '-': + continue + if not os.path.exists(arg): + print("'%s' not found" % arg) + sys.exit(1) + for arg in self.args: + if arg == '-': + self.hexanonymize(sys.stdin) + else: + with open(arg) as filehandle: + self.hexanonymize(filehandle) + + +if __name__ == '__main__': + HexAnonymize().main() From 56ce78a2273edb76dfeaa068fbd2bf6de723fdaf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Jan 2020 18:07:53 +0000 Subject: [PATCH 0599/2779] updated hexanonymize.py --- hexanonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hexanonymize.py b/hexanonymize.py index 600083615..de037485e 100755 --- a/hexanonymize.py +++ b/hexanonymize.py @@ -84,13 +84,13 @@ def hexanonymize(self, filehandle): integer += 1 if integer > 9: integer = 0 - elif char.lower() in hex_alphas or (not only_hex_alphas and char.isalpha()): + elif (not only_hex_alphas and char.isalpha()) or char.lower() in hex_alphas: if preserve_case and char.isupper(): char = letter.upper() else: char = letter letter = chr(ord(char) + 1) - if letter not in hex_alphas: + if letter.lower() not in hex_alphas: letter = 'a' print(char, end='') From 267634f4773ff3191f1fd61e90b8fd7ac6f620cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Jan 2020 18:08:37 +0000 Subject: [PATCH 0600/2779] added test_hexanonymize.sh --- tests/test_hexanonymize.sh | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 tests/test_hexanonymize.sh diff --git a/tests/test_hexanonymize.sh b/tests/test_hexanonymize.sh new file mode 100755 index 000000000..57da1e297 --- /dev/null +++ b/tests/test_hexanonymize.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-01-02 17:35:08 +0000 (Thu, 02 Jan 2020) +# +# https://github.com/harisekhon/devop-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# + +set -eu +[ -n "${DEBUG:-}" ] && set -x +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd "$srcdir/.."; + +. ./tests/utils.sh + +section "HexAnonymize" + +start_time=$(date +%s) + +run++ +check_output "abc123456def789012abcd" hexanonymize.py <<< "xyz987654rst654321AKIA" + +run++ +check_output "abc123456def789012ABCD" hexanonymize.py -c <<< "xyz987654rst654321AKIA" + +run++ +check_output "xyz123456rst789012abC" hexanonymize.py -c -o <<< "xyz987654rst654321caD" + +run++ +check_output "xyz123456rst789012abc" hexanonymize.py -o <<< "xyz987654rst654321caD" + +echo +echo "Total Tests run: $run_count" +time_taken "$start_time" "All version tests for $name completed in" +echo +untrap From c50eebeba4c84f641d4d48c6aa75c01baa876299 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Jan 2020 18:12:03 +0000 Subject: [PATCH 0601/2779] updated hexanonymize.py --- hexanonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hexanonymize.py b/hexanonymize.py index de037485e..91728e7fc 100755 --- a/hexanonymize.py +++ b/hexanonymize.py @@ -63,13 +63,13 @@ def __init__(self): def add_options(self): super(HexAnonymize, self).add_options() self.add_opt('-c', '--case', action='store_true', help='Preserve case') - self.add_opt('-o', '--only-hex', action='store_true', + self.add_opt('-o', '--hex-only', action='store_true', help='Only replace hex alpha chars (A-F, a-f), otherwise replaces all alphanumerics for safety') def process_options(self): super(HexAnonymize, self).process_options() self.preserve_case = self.get_opt('case') - self.only_hex_alphas = self.get_opt('only_hex') + self.only_hex_alphas = self.get_opt('hex_only') def hexanonymize(self, filehandle): preserve_case = self.preserve_case From 88a6f2b6054294da428b680c288ae7acdf7f49e4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 3 Jan 2020 23:26:35 +0000 Subject: [PATCH 0602/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 53c641477..8c9498024 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -442,26 +442,26 @@ dest[115]="arn:aws:iam:::user//*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" dest[116]="arn:aws:iam:::group/*" -src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" -dest[116]="arn:aws:iam:::group//*" +src[117]="arn:aws:iam::123456789012:group/Development/product_1234/*" +dest[117]="arn:aws:iam:::group//*" -src[117]="arn:aws:s3:::my_corporate_bucket/Development/*" -dest[117]="arn:aws:s3:::*" +src[118]="arn:aws:s3:::my_corporate_bucket/Development/*" +dest[118]="arn:aws:s3:::*" -src[118]="AKIAIOSFODNN7EXAMPLE" -dest[118]="" +src[119]="AKIAIOSFODNN7EXAMPLE" +dest[119]="" -src[119]="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" -dest[119]="" +src[120]="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +dest[120]="" -src[120]="AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE" -dest[120]="" +src[121]="AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE" +dest[121]="" -src[121]="ASIAIOSFODNN7EXAMPLE" -dest[121]="" +src[122]="ASIAIOSFODNN7EXAMPLE" +dest[122]="" -src[122]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" -dest[122]="" +src[123]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" +dest[123]="" # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 2efd6307651985eda772a23663cd73fd17e78229 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:28:12 +0000 Subject: [PATCH 0603/2779] renamed hadoop_hdfs_files_native_checksums.jy to hdfs_files_native_checksums.jy --- ...fs_files_native_checksums.jy => hdfs_files_native_checksums.jy | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hadoop_hdfs_files_native_checksums.jy => hdfs_files_native_checksums.jy (100%) diff --git a/hadoop_hdfs_files_native_checksums.jy b/hdfs_files_native_checksums.jy similarity index 100% rename from hadoop_hdfs_files_native_checksums.jy rename to hdfs_files_native_checksums.jy From 3a00a1e52f75c2948ad0b8cdf2736c7a99062196 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:28:18 +0000 Subject: [PATCH 0604/2779] renamed hadoop_hdfs_files_stats.jy to hdfs_files_stats.jy --- hadoop_hdfs_files_stats.jy => hdfs_files_stats.jy | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hadoop_hdfs_files_stats.jy => hdfs_files_stats.jy (100%) diff --git a/hadoop_hdfs_files_stats.jy b/hdfs_files_stats.jy similarity index 100% rename from hadoop_hdfs_files_stats.jy rename to hdfs_files_stats.jy From 25725f41def7c2188b2bb0fd3b75aefa10ecb45e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:28:23 +0000 Subject: [PATCH 0605/2779] renamed hadoop_hdfs_time_block_reads.jy to hdfs_time_block_reads.jy --- hadoop_hdfs_time_block_reads.jy => hdfs_time_block_reads.jy | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hadoop_hdfs_time_block_reads.jy => hdfs_time_block_reads.jy (100%) diff --git a/hadoop_hdfs_time_block_reads.jy b/hdfs_time_block_reads.jy similarity index 100% rename from hadoop_hdfs_time_block_reads.jy rename to hdfs_time_block_reads.jy From 604f0e9c86e37223b484c4e65b400fa21e3ca93e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:30:16 +0000 Subject: [PATCH 0606/2779] updated README.md --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e316eba4e..9d75612da 100644 --- a/README.md +++ b/README.md @@ -105,9 +105,10 @@ Environment variables are supported for convenience and also to hide credentials - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - ```hadoop_hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - ```hadoop_hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - ```hdfs_find_replication_factor_1.py``` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows + - ```hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - ```hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex @@ -257,14 +258,14 @@ make The 3 Hadoop utility programs listed below require Jython (as well as Hadoop to be installed and correctly configured) ``` -hadoop_hdfs_time_block_reads.jy -hadoop_hdfs_files_native_checksums.jy -hadoop_hdfs_files_stats.jy +hdfs_time_block_reads.jy +hdfs_files_native_checksums.jy +hdfs_files_stats.jy ``` Run like so: ``` -jython -J-cp $(hadoop classpath) hadoop_hdfs_time_block_reads.jy --help +jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` The ```-J-cp $(hadoop classpath) ``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. @@ -286,7 +287,7 @@ Jython is a simple download and unpack and can be fetched from http://www.jython Then add the Jython install bin directory to the $PATH or specify the full path to the `jython` binary, eg: ``` -/opt/jython-2.7.0/bin/jython hadoop_hdfs_time_block_reads.jy ... +/opt/jython-2.7.0/bin/jython hdfs_time_block_reads.jy ... ``` From 1db035361637922ed487f80be6e4313d3eb4253a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Jan 2020 17:31:20 +0000 Subject: [PATCH 0607/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d75612da..b063a839a 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,7 @@ Run like so: jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` -The ```-J-cp $(hadoop classpath) ``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. +The ```-J-cp $(hadoop classpath)``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. See below for procedure to install Jython if you don't already have it. From 1577b0319456095a477676b8124cdc3063a687dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 8 Jan 2020 17:49:07 +0000 Subject: [PATCH 0608/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 8c9498024..c18db1ae6 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -440,7 +440,7 @@ src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*" dest[115]="arn:aws:iam:::user//*" src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*" -dest[116]="arn:aws:iam:::group/*" +dest[116]="arn:aws:iam:::group//*" src[117]="arn:aws:iam::123456789012:group/Development/product_1234/*" dest[117]="arn:aws:iam:::group//*" @@ -463,6 +463,9 @@ dest[122]="" src[123]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" dest[123]="" +src[124]="sg-5f63c627" +dest[124]="" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 8573959780d35723f0324f05719a96e9bdabf1fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 8 Jan 2020 17:59:53 +0000 Subject: [PATCH 0609/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index c18db1ae6..c1fa8450e 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -466,6 +466,9 @@ dest[123]="" src[124]="sg-5f63c627" dest[124]="" +src[125]="s3://myBucket/file.txt" +dest[125]="s3:///file.txt" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 34e192dc078bef5112dc6530764917548fb0746f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 8 Jan 2020 18:00:21 +0000 Subject: [PATCH 0610/2779] added aws security group and bucket anonymization --- anonymize.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 15e54eb4c..99e1dd31c 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.0' +__version__ = '0.10.1' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -296,6 +296,8 @@ def __init__(self): 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key + 'aws7': r'\bsg-[a-z0-9]{8}(?!\w)', # security group id + 'aws8': r'(\bs3a?)://[^/]+/', # s3 bucket name # don't change hostname or fqdn regex without updating hash_hostnames() option parse # since that replaces these replacements and needs to match the grouping captures and surrounding format 'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex), @@ -407,6 +409,8 @@ def __init__(self): 'aws4': r'', 'aws5': r'', 'aws6': r'', + 'aws7': r'', + 'aws8': r'\1:///', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', From b7867276c5ebe0813e04d4237f91c02e5588adaa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 10:27:26 +0000 Subject: [PATCH 0611/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index c1fa8450e..c3c0bcd54 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -463,12 +463,26 @@ dest[122]="" src[123]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==" dest[123]="" +# security groups src[124]="sg-5f63c627" -dest[124]="" +dest[124]="" src[125]="s3://myBucket/file.txt" dest[125]="s3:///file.txt" +# RDS +src[126]="--db-name myDB" +dest[126]="--db-name " + +src[127]="--db-instance-identifier myDBinstance" +dest[127]="--db-instance-identifier " + +src[128]="--master-user-password blah" +dest[128]="--master-user-password " + +src[129]="--master-username first.last" +dest[129]="--master-username " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From d74d2a0c9bb7b3e5be7c94b64454dee6b31a44eb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 10:28:19 +0000 Subject: [PATCH 0612/2779] added --database, change aws security group and cisco password, improved -password matches --- anonymize.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/anonymize.py b/anonymize.py index 99e1dd31c..5c28bf27f 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.1' +__version__ = '0.10.2' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -118,6 +118,7 @@ def __init__(self): ('ip', False), ('subnet_mask', False), ('mac', False), + ('db', False), ('kerberos', False), ('email', False), ('password', False), @@ -296,8 +297,10 @@ def __init__(self): 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key - 'aws7': r'\bsg-[a-z0-9]{8}(?!\w)', # security group id + 'aws7': r'\bsg-[A-Za-z0-9]{8}(?).*?$', 'cisco3': r'\ssecret\s.*?$', 'cisco4': r'\smd5\s+.*?$', 'cisco5': r'\scommunity\s+.*$', @@ -409,8 +412,10 @@ def __init__(self): 'aws4': r'', 'aws5': r'', 'aws6': r'', - 'aws7': r'', + 'aws7': r'', 'aws8': r'\1:///', + 'db': r'\1', + 'db2': r'\1', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', @@ -489,7 +494,10 @@ def add_options(self): self.add_opt('-a', '--all', action='store_true', help='Apply all anonymizations (careful this includes --host which can be overzealous and ' + \ 'match too many things, in which case try more targeted anonymizations below)') - self.add_opt('-w', '--aws', action='store_true', help='Apply AWS ARN anonymizations') + self.add_opt('-w', '--aws', action='store_true', + help='Apply AWS anonymizations (access/secret keys, STS tokens, ARNs, buckets, security groups)') + self.add_opt('-b', '--db', '--database', action='store_true', + help='Apply database anonymizations (db name, instance name)') self.add_opt('-C', '--custom', action='store_true', help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \ 'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \ @@ -600,7 +608,10 @@ def process_options(self): for _ in self.anonymizations: if _ in ('subnet_mask', 'mac', 'group'): continue - self.anonymizations[_] = self.get_opt(_) + elif _ == 'database': + self.anonymizations['db'] = True + else: + self.anonymizations[_] = self.get_opt(_) log.debug('anonymization enabled %s = %s', _, bool(self.anonymizations[_])) self._process_options_host() self._process_options_network() From 5f3458919739c6eb85412f260fe49664273deea8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 13:28:21 +0000 Subject: [PATCH 0613/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index c3c0bcd54..1de505f57 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -483,6 +483,9 @@ dest[128]="--master-user-password " src[129]="--master-username first.last" dest[129]="--master-username " +src[130]="--schema-name mySchema" +dest[130]="--schema-name " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From a8c25d519e600f0738e1a550fa4b4efde39266db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 13:28:27 +0000 Subject: [PATCH 0614/2779] updated anonymize.py --- anonymize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/anonymize.py b/anonymize.py index 5c28bf27f..ac639541a 100755 --- a/anonymize.py +++ b/anonymize.py @@ -301,6 +301,7 @@ def __init__(self): 'aws8': r'(\bs3a?)://[^/]+/', # s3 bucket name 'db': r'(-{1,2}(?:db|database)?-?name' + r'{arg_sep})\S+'.format(arg_sep=arg_sep), 'db2': r'(-{1,2}(?:db|database)-?instance(-?identifier)?' + r'{arg_sep})\S+'.format(arg_sep=arg_sep), + 'db3': r'(-{1,2}schema-?name' + r'{arg_sep})\S+'.format(arg_sep=arg_sep), # don't change hostname or fqdn regex without updating hash_hostnames() option parse # since that replaces these replacements and needs to match the grouping captures and surrounding format 'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex), @@ -416,6 +417,7 @@ def __init__(self): 'aws8': r'\1:///', 'db': r'\1', 'db2': r'\1', + 'db3': r'\1', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', From 122fdb64fad622d70c35c5f63ddb50f92201e5b1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 13:30:05 +0000 Subject: [PATCH 0615/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 1de505f57..c55d8637c 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -486,6 +486,9 @@ dest[129]="--master-username " src[130]="--schema-name mySchema" dest[130]="--schema-name " +src[131]="=arn:aws:acm:us-east-1:123456:certificate/abc-123" +dest[131]="=arn:aws:acm:us-east-1::certificate/" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 17bb20ef8a82827bb93675bea3dc9cc84f810718 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 14:00:24 +0000 Subject: [PATCH 0616/2779] added -key match and improved db regex --- anonymize.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index ac639541a..0214065e6 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.2' +__version__ = '0.10.3' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -299,9 +299,10 @@ def __init__(self): 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key 'aws7': r'\bsg-[A-Za-z0-9]{8}(?', 'aws7': r'', 'aws8': r'\1:///', + 'aws9': r'\1', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', From a0b35853a6cc6afec2ca1580454c7052f33d3ca3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 14:00:28 +0000 Subject: [PATCH 0617/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index c55d8637c..70d331acd 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -489,6 +489,12 @@ dest[130]="--schema-name " src[131]="=arn:aws:acm:us-east-1:123456:certificate/abc-123" dest[131]="=arn:aws:acm:us-east-1::certificate/" +src[132]="--key-name my-key" +dest[132]="--key-name " + +src[133]="private-key my-key" +dest[133]="private-key " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" #dest[103]="proxy port " From 26b27dba68b5f4993b3aaaefc70e1a08c87969c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 14:36:55 +0000 Subject: [PATCH 0618/2779] updated anonymize.py --- anonymize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index 0214065e6..c610901c9 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.3' +__version__ = '0.10.4' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -300,9 +300,9 @@ def __init__(self): 'aws7': r'\bsg-[A-Za-z0-9]{8}(? Date: Thu, 9 Jan 2020 15:10:40 +0000 Subject: [PATCH 0619/2779] refined switch prefix variant, added --cluster variants --- anonymize.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index c610901c9..eafca0e97 100755 --- a/anonymize.py +++ b/anonymize.py @@ -283,6 +283,8 @@ def __init__(self): arg_sep = r'[=\s:]+' # openssl uses -passin switch pass_word_phrase = r'(?:pass(?:word|phrase|in)?|userPassword)' + # allowing --blah- prefix variants + switch_prefix = r'(?', 'aws8': r'\1:///', 'aws9': r'\1', + 'aws10': r'\1', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', From 777f5bdcf383c8281a4df99c99e97a926c2b9814 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 15:10:43 +0000 Subject: [PATCH 0620/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 70d331acd..1dce824c7 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -470,12 +470,11 @@ dest[124]="" src[125]="s3://myBucket/file.txt" dest[125]="s3:///file.txt" -# RDS -src[126]="--db-name myDB" -dest[126]="--db-name " +src[126]="aws rds create-db-instance --db-name myDB" +dest[126]="aws rds create-db-instance --db-name " -src[127]="--db-instance-identifier myDBinstance" -dest[127]="--db-instance-identifier " +src[127]="aws rds modify-db-instance --db-instance-identifier myDBinstance" +dest[127]="aws rds modify-db-instance --db-instance-identifier " src[128]="--master-user-password blah" dest[128]="--master-user-password " @@ -492,8 +491,12 @@ dest[131]="=arn:aws:acm:us-east-1::certificate/" src[132]="--key-name my-key" dest[132]="--key-name " -src[133]="private-key my-key" -dest[133]="private-key " +src[133]="-private-key my-key" +dest[133]="-private-key " + +src[134]="aws elasticache create-cache-cluster --cache-cluster-id myCluster" +dest[134]="aws elasticache create-cache-cluster --cache-cluster-id " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From b6702dd2a2dd0ae77720cbb6fd7aaeabfe2ac144 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 18:08:40 +0000 Subject: [PATCH 0621/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 1dce824c7..0fe2ee62c 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -497,6 +497,9 @@ dest[133]="-private-key " src[134]="aws elasticache create-cache-cluster --cache-cluster-id myCluster" dest[134]="aws elasticache create-cache-cluster --cache-cluster-id " +src[135]="subnet-abc12345" +dest[135]="" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 7cfc78cd5658b084d376ddf42143431603efc9ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 9 Jan 2020 18:09:12 +0000 Subject: [PATCH 0622/2779] updated anonymize.py --- anonymize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/anonymize.py b/anonymize.py index eafca0e97..637c9b280 100755 --- a/anonymize.py +++ b/anonymize.py @@ -303,6 +303,7 @@ def __init__(self): 'aws8': r'(\bs3a?)://[^/]+/', # s3 bucket name 'aws9': r'({switch_prefix}key(:?-?name)?{arg_sep})[\w-]+'.format(arg_sep=arg_sep, switch_prefix=switch_prefix), 'aws10': r'({switch_prefix}cluster(?:-?id)?{arg_sep})[\w-]+'.format(arg_sep=arg_sep, switch_prefix=switch_prefix), + 'aws11': r'(?/', 'aws9': r'\1', 'aws10': r'\1', + 'aws11': r'', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', From 3c89b46e3a41388333122904769b7cd234063c31 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 10:05:50 +0000 Subject: [PATCH 0623/2779] updated anonymize.py --- anonymize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 637c9b280..28a1c2bcd 100755 --- a/anonymize.py +++ b/anonymize.py @@ -303,7 +303,7 @@ def __init__(self): 'aws8': r'(\bs3a?)://[^/]+/', # s3 bucket name 'aws9': r'({switch_prefix}key(:?-?name)?{arg_sep})[\w-]+'.format(arg_sep=arg_sep, switch_prefix=switch_prefix), 'aws10': r'({switch_prefix}cluster(?:-?id)?{arg_sep})[\w-]+'.format(arg_sep=arg_sep, switch_prefix=switch_prefix), - 'aws11': r'(? Date: Fri, 10 Jan 2020 10:09:12 +0000 Subject: [PATCH 0624/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 0fe2ee62c..878cac882 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -500,6 +500,9 @@ dest[134]="aws elasticache create-cache-cluster --cache-cluster-id " src[135]="subnet-abc12345" dest[135]="" +src[136]="arn:aws:acm:us-east-1:123456:function:myFunction123:7" +dest[136]="arn:aws:acm:us-east-1::function::7" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 4dd0868fc84c9ff485f7f3c9d73474c0abce5ba7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 13:33:06 +0000 Subject: [PATCH 0625/2779] added --generic --- anonymize.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/anonymize.py b/anonymize.py index 28a1c2bcd..f5db7e4b5 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.4' +__version__ = '0.10.5' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -119,6 +119,7 @@ def __init__(self): ('subnet_mask', False), ('mac', False), ('db', False), + ('generic', False), ('kerberos', False), ('email', False), ('password', False), @@ -285,6 +286,7 @@ def __init__(self): pass_word_phrase = r'(?:pass(?:word|phrase|in)?|userPassword)' # allowing --blah- prefix variants switch_prefix = r'(?', 'aws7': r'', 'aws8': r'\1:///', - 'aws9': r'\1', - 'aws10': r'\1', - 'aws11': r'', + 'aws9': r'', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', + 'generic': r'\1://', + 'generic2': r'\1', + 'generic3': r'\1', + 'generic4': r'\1', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', @@ -508,6 +514,8 @@ def add_options(self): help='Apply AWS anonymizations (access/secret keys, STS tokens, ARNs, buckets, security groups)') self.add_opt('-b', '--db', '--database', action='store_true', help='Apply database anonymizations (db name, instance name)') + self.add_opt('-g', '--generic', action='store_true', + help='Apply generic anonymizations (file://, key, cluster name / id etc)') self.add_opt('-C', '--custom', action='store_true', help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \ 'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \ From 30245833319b83af58e0798cdba47b9eace1b596 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 13:33:11 +0000 Subject: [PATCH 0626/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 878cac882..1717579b1 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -503,6 +503,9 @@ dest[135]="" src[136]="arn:aws:acm:us-east-1:123456:function:myFunction123:7" dest[136]="arn:aws:acm:us-east-1::function::7" +src[137]="aws lambda update-function-code --function-name hari-test --zip-file fileb://myfunction.zip" +dest[137]="aws lambda update-function-code --function-name --zip-file fileb://" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 32854e2911b8c1becc16d0e2faf2e3c88d77be85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 15:11:42 +0000 Subject: [PATCH 0627/2779] updated anonymize.py --- anonymize.py | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/anonymize.py b/anonymize.py index f5db7e4b5..2f7fb8a21 100755 --- a/anonymize.py +++ b/anonymize.py @@ -120,6 +120,8 @@ def __init__(self): ('mac', False), ('db', False), ('generic', False), + # kerberos must be applied before email + # - if email is applied first, 'user/host@realm' becomes 'user/', exposing user ('kerberos', False), ('email', False), ('password', False), @@ -304,13 +306,30 @@ def __init__(self): 'aws7': r'\bsg-[A-Za-z0-9]{8}(? ' + \ r'http://:\@. Also works with https://') self.add_opt('-K', '--kerberos', action='store_true', - help=r'Kerberos 5 principals in the form @ or /@ ' + \ + help=r'Apply Kerberos anonymizations eg. @, /@ ' + \ '(where must match a valid domain name - otherwise use --custom and populate ' + \ - r'anonymize_custom.conf). These kerberos principals are anonymizebed to ' + \ - '. There is a special exemption for Hadoop Kerberos principals such ' + \ - 'as NN/_HOST@ which preserves the literal \'_HOST\' instance since that\'s ' + \ - 'useful to know for debugging, the principal and realm will still be anonymizebed in ' + \ - 'those cases (if wanting to retain NN/_HOST then use --domain instead of --kerberos). ' + \ - 'This is applied before --email in order to not prevent the email replacement leaving ' + \ - r'this as user/host\@realm to user/, which would have exposed \'user\'' + \ - '. Auto enables --email, --domain and --fqdn') + r'anonymize_custom.conf). Hadoop principals preserve the generic _HOST placeholder eg. ' + \ + '/_HOST@ (if wanting to retain full prefix eg. NN/_HOST then use ' + \ + '--domain instead of --kerberos). --kerberos auto-enables --email, --domain and --fqdn') self.add_opt('-L', '--ldap', action='store_true', help='Apply LDAP anonymization ' + \ '(~100 attribs eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...)') @@ -583,7 +597,7 @@ def add_options(self): # 'should probably also apply --ip and --host if using this. Auto enables --http-auth') self.add_opt('-N', '--network', action='store_true', help='Apply all network anonymization, whether Cisco, ScreenOS, JunOS for secrets, auth, ' + \ - 'usernames, passwords, md5s, PSKs, AS, SNMP etc.') + 'usernames, passwords, md5s, PSKs, AS, SNMP community strings etc.') self.add_opt('-c', '--cisco', action='store_true', help='Apply Cisco IOS/IOS-XR/NX-OS configuration format anonymization') self.add_opt('-s', '--screenos', action='store_true', From e5357fd74400c98d4f747c13bbcbe70f72489632 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 10 Jan 2020 15:14:32 +0000 Subject: [PATCH 0628/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b063a839a..17efe0c68 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Environment variables are supported for convenience and also to hide credentials - Linux: - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) - - anonymizes: + - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses - IP + MAC addresses From f0bdacc587ff59355f5526c77dbf6652200ffbaa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 10:07:14 +0000 Subject: [PATCH 0629/2779] updated anonymize.py --- anonymize.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/anonymize.py b/anonymize.py index 2f7fb8a21..8210af2d6 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.5' +__version__ = '0.10.6' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -318,15 +318,19 @@ def __init__(self): id_or_name=id_or_name, switch_prefix=switch_prefix), 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex), - 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})[\w-]+'\ + 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), - 'generic3': r'({switch_prefix}cluster{id_or_name}?{arg_sep})[\w-]+'\ + 'generic3': r'({switch_prefix}cluster{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), - 'generic4': r'({switch_prefix}function{id_or_name}?{arg_sep})[\w-]+'\ + 'generic4': r'({switch_prefix}function{id_or_name}?{arg_sep})\S+'\ + .format(arg_sep=arg_sep, + id_or_name=id_or_name, + switch_prefix=switch_prefix), + 'generic5': r'({switch_prefix}load-?balancer{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), @@ -451,6 +455,7 @@ def __init__(self): 'generic2': r'\1', 'generic3': r'\1', 'generic4': r'\1', + 'generic5': r'\1', 'hostname': r':\2', #'hostname2': '', 'hostname2': r'', From 55f6afc7e430bd085fb131ddda87c89e4c250763 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 10:07:32 +0000 Subject: [PATCH 0630/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 1717579b1..b203357f4 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -506,6 +506,9 @@ dest[136]="arn:aws:acm:us-east-1::function::7" src[137]="aws lambda update-function-code --function-name hari-test --zip-file fileb://myfunction.zip" dest[137]="aws lambda update-function-code --function-name --zip-file fileb://" +src[138]=' aws elb create-load-balancer --load-balancer-name "$lb_name" ...' +dest[138]=' aws elb create-load-balancer --load-balancer-name ...' + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 08eca37ea8184589d9416d1002a704489f944d1d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 10:23:49 +0000 Subject: [PATCH 0631/2779] added aws_users_unused_access_keys.py --- aws_users_unused_access_keys.py | 151 ++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 aws_users_unused_access_keys.py diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py new file mode 100755 index 000000000..00440214d --- /dev/null +++ b/aws_users_unused_access_keys.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019) +# +# https://github.com/harisekhon/nagios-plugins +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Find AWS IAM user access keys that haven't been used in N days or since created + +Generates an IAM credential report, then parses it to determine the time since each user's password +and access keys were last used + +Requires iam:GenerateCredentialReport on resource: * + +Output: + + + + +Uses the Boto python library, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html + +See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools + +- https://github.com/harisekhon/devops-bash-tools +- https://github.com/harisekhon/nagios-plugins + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import csv +import os +import sys +import time +import traceback +from datetime import datetime +from io import StringIO +from math import floor +import boto3 +from botocore.exceptions import ClientError +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_int + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class AWSUnusedAccessKeys(CLI): + + def __init__(self): + # Python 2.x + super(AWSUnusedAccessKeys, self).__init__() + # Python 3.x + # super().__init__() + self.age = None + self.now = None + self.timeout_default = 300 + self.msg = 'AWSUnusedAccessKeys msg not defined' + + def add_options(self): + self.add_opt('-a', '--age', type=float, default=90, + help='Show only access keys not used in the last N days (default: 90)') + + def process_args(self): + self.no_args() + self.age = self.get_opt('age') + if self.age is not None: + validate_int(self.age, 'age') + + def run(self): + iam = boto3.client('iam') + log.info('generating credentials report') + while True: + result = iam.generate_credential_report() + log.debug('%s', result) + if result['State'] == 'COMPLETE': + log.info('credentials report generated') + break + log.info('waiting for credentials report') + time.sleep(1) + try: + result = iam.get_credential_report() + except ClientError as _: + raise + csv_content = result['Content'] + log.debug('%s', csv_content) + filehandle = StringIO(unicode(csv_content)) + filehandle.seek(0) + csvreader = csv.reader(filehandle) + headers = csvreader.next() + log.debug('headers: %s', headers) + assert headers[0] == 'user' + assert headers[4] == 'password_last_used' + assert headers[10] == 'access_key_1_last_used_date' + assert headers[15] == 'access_key_2_last_used_date' + self.now = datetime.utcnow() + for row in csvreader: + self.process_user(row) + + def process_user(self, row): + log.debug('processing user: %s', row) + user = row[0] + password_last_used = row[4] + access_key_1_last_used_date = row[10] + access_key_2_last_used_date = row[15] + log.debug('user: %s, password_last_used: %s, access_key_1_last_used_date: %s, access_key_2_last_used_date: %s', + user, password_last_used, access_key_1_last_used_date, access_key_2_last_used_date) + key = 1 + for _ in [access_key_1_last_used_date, access_key_2_last_used_date]: + if _ == 'N/A': + continue + # %z not working in Python 2.7 but we already know it's +00:00 + _datetime = datetime.strptime(_.split('+')[0], '%Y-%m-%dT%H:%M:%S') + age_timedelta = self.now - _datetime.replace(tzinfo=None) + age_days = int(floor(age_timedelta.total_seconds() / 86400.0)) + if age_days > self.age: + log.debug('access key %s last used %s days ago > %s', key, age_days, self.age) + print('{user:20}\t{key}\t{days:>3}\t{access_key_last_used:25}\t'\ + .format(user=user, + key=key, + days=age_days, + access_key_last_used=_)) + key += 1 + + +if __name__ == '__main__': + AWSUnusedAccessKeys().main() From 9b70c9638f194e1ca8c86baca6670b85057a1ab2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 10:25:44 +0000 Subject: [PATCH 0632/2779] updated aws_users_unused_access_keys.py --- aws_users_unused_access_keys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py index 00440214d..b7fcdf9cd 100755 --- a/aws_users_unused_access_keys.py +++ b/aws_users_unused_access_keys.py @@ -114,7 +114,6 @@ def run(self): headers = csvreader.next() log.debug('headers: %s', headers) assert headers[0] == 'user' - assert headers[4] == 'password_last_used' assert headers[10] == 'access_key_1_last_used_date' assert headers[15] == 'access_key_2_last_used_date' self.now = datetime.utcnow() From 223e005b504a4d89698ed09fba5c7b4102d2b5f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 11:14:22 +0000 Subject: [PATCH 0633/2779] updated aws_users_unused_access_keys.py --- aws_users_unused_access_keys.py | 76 +++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py index b7fcdf9cd..9b9083eb9 100755 --- a/aws_users_unused_access_keys.py +++ b/aws_users_unused_access_keys.py @@ -16,7 +16,7 @@ """ -Find AWS IAM user access keys that haven't been used in N days or since created +Find AWS IAM user access keys that haven't been used in N days and keys that have never been used Generates an IAM credential report, then parses it to determine the time since each user's password and access keys were last used @@ -25,8 +25,7 @@ Output: - - + Uses the Boto python library, read here for the list of ways to configure your AWS credentials: @@ -66,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' class AWSUnusedAccessKeys(CLI): @@ -78,12 +77,15 @@ def __init__(self): # super().__init__() self.age = None self.now = None + self.only_active = False self.timeout_default = 300 self.msg = 'AWSUnusedAccessKeys msg not defined' def add_options(self): - self.add_opt('-a', '--age', type=float, default=90, - help='Show only access keys not used in the last N days (default: 90)') + self.add_opt('-a', '--age', type=float, default=30, + help='Show only access keys not used in the last N days (default: 30)') + self.add_opt('-o', '--only-active', action='store_true', + help='Only show access keys that are active') def process_args(self): self.no_args() @@ -114,7 +116,11 @@ def run(self): headers = csvreader.next() log.debug('headers: %s', headers) assert headers[0] == 'user' + assert headers[8] == 'access_key_1_active' + assert headers[9] == 'access_key_1_last_rotated' assert headers[10] == 'access_key_1_last_used_date' + assert headers[13] == 'access_key_2_active' + assert headers[14] == 'access_key_2_last_rotated' assert headers[15] == 'access_key_2_last_used_date' self.now = datetime.utcnow() for row in csvreader: @@ -123,27 +129,53 @@ def run(self): def process_user(self, row): log.debug('processing user: %s', row) user = row[0] - password_last_used = row[4] - access_key_1_last_used_date = row[10] - access_key_2_last_used_date = row[15] - log.debug('user: %s, password_last_used: %s, access_key_1_last_used_date: %s, access_key_2_last_used_date: %s', - user, password_last_used, access_key_1_last_used_date, access_key_2_last_used_date) - key = 1 - for _ in [access_key_1_last_used_date, access_key_2_last_used_date]: - if _ == 'N/A': + access_keys = {1:{}, 2:{}} + access_keys[1]['active'] = row[8] + access_keys[1]['last_used_date'] = row[10] + access_keys[1]['last_rotated'] = row[9] + access_keys[2]['active'] = row[13] + access_keys[2]['last_rotated'] = row[14] + access_keys[2]['last_used_date'] = row[15] + for key in [1, 2]: + active = access_keys[key]['active'] + if not isinstance(active, bool): + assert active in ('true', 'false') + active = active.lower() == 'true' + created = access_keys[key]['last_rotated'] + last_used = access_keys[key]['last_used_date'] + log.debug('user: %s, key: %s, active: %s, created: %s, last_used_date: %s, ', + user, + key, + active, + created, + last_used + ) + if not active and self.only_active: + continue + if last_used == 'N/A': + if created == 'N/A': + continue + self.print_key(user, key, active, 'N/A', last_used, created) continue # %z not working in Python 2.7 but we already know it's +00:00 - _datetime = datetime.strptime(_.split('+')[0], '%Y-%m-%dT%H:%M:%S') + _datetime = datetime.strptime(last_used.split('+')[0], '%Y-%m-%dT%H:%M:%S') age_timedelta = self.now - _datetime.replace(tzinfo=None) age_days = int(floor(age_timedelta.total_seconds() / 86400.0)) if age_days > self.age: - log.debug('access key %s last used %s days ago > %s', key, age_days, self.age) - print('{user:20}\t{key}\t{days:>3}\t{access_key_last_used:25}\t'\ - .format(user=user, - key=key, - days=age_days, - access_key_last_used=_)) - key += 1 + self.print_key(user, key, active, age_days, last_used, created) + + # pylint: disable=too-many-arguments + def print_key(self, user, key, active, age_days, last_used, created): + log.debug('access key %s, active: %s, last used %s days ago > %s', key, active, age_days, self.age) + print('{user:20}\t{key}\t{active}\t{days:>3}\t{last_used:25}\t{created}'\ + .format(user=user, + key=key, + active='Active' if active else 'Inactive', + days=age_days, + last_used=last_used, + created=created + ) + ) if __name__ == '__main__': From 1cee9e5977c85848a0d323061c00abe6b43a1c87 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 13 Jan 2020 11:16:46 +0000 Subject: [PATCH 0634/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 17efe0c68..b7f45f612 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,8 @@ Environment variables are supported for convenience and also to hide credentials - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [AWS](https://aws.amazon.com/): - - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days + - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) + - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Hadoop](http://hadoop.apache.org/) & NoSQL: From debb9721fae59e57a204f4ee14dbb6a683b8d1d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 14 Jan 2020 11:46:43 +0000 Subject: [PATCH 0635/2779] updated aws_users_access_key_age.py --- aws_users_access_key_age.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 9d39caf33..c0026570d 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -24,6 +24,8 @@ Status is Active or Inactive +Validated compared to xls report download from Trusted Advisor -> Security -> IAM Access Key Rotation + Uses the Boto library, read here for the list of ways to configure your AWS credentials: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html From 60fa757168a15138dd1c4ce2be1cfc9f7f678ab0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 14 Jan 2020 17:56:56 +0000 Subject: [PATCH 0636/2779] added aws_s3_presign_url.py --- aws_s3_presign_url.py | 67 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100755 aws_s3_presign_url.py diff --git a/aws_s3_presign_url.py b/aws_s3_presign_url.py new file mode 100755 index 000000000..c48a6147c --- /dev/null +++ b/aws_s3_presign_url.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-01-14 17:45:38 +0000 (Tue, 14 Jan 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Generate and print an S3 pre-signed URL + +Will generate a pre-signed URL even when the bucket and object key don't exist! + +(you will get a runtime error when requesting the link that the bucket or object doesn't exist) + + +Uses the Boto library, read here for the list of ways to configure your AWS credentials: + + https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import boto3 + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +def main(): + parser = argparse.ArgumentParser( + description='Generate an AWS S3 pre-signed URL to access an S3 object without logging in') + parser.add_argument('bucket', help='Bucket Name') + parser.add_argument('key', help='Key') + parser.add_argument('expiration', nargs='?', default=3600, help='Expiration of URL in seconds') + args = parser.parse_args() + + conn = boto3.client('s3') + url = conn.generate_presigned_url( + 'get_object', + Params={ + 'Bucket': args.bucket, + 'Key': args.key + }, + ExpiresIn=args.expiration + ) + print(url) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print('Control-C...') From edace9cf841cedcb7203edca90b4e44c3a9b0b8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 14 Jan 2020 20:33:36 +0000 Subject: [PATCH 0637/2779] updated aws_s3_presign_url.py --- aws_s3_presign_url.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aws_s3_presign_url.py b/aws_s3_presign_url.py index c48a6147c..57f4d965d 100755 --- a/aws_s3_presign_url.py +++ b/aws_s3_presign_url.py @@ -35,10 +35,11 @@ from __future__ import unicode_literals import argparse +import sys import boto3 __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' def main(): parser = argparse.ArgumentParser( @@ -64,4 +65,4 @@ def main(): try: main() except KeyboardInterrupt: - print('Control-C...') + print('Control-C...', file=sys.stderr) From 4e498e905c2547236d7a2baef2b7f23d727b0749 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 10:20:49 +0000 Subject: [PATCH 0638/2779] updated aws_s3_presign_url.py --- aws_s3_presign_url.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aws_s3_presign_url.py b/aws_s3_presign_url.py index 57f4d965d..2fbb1331f 100755 --- a/aws_s3_presign_url.py +++ b/aws_s3_presign_url.py @@ -18,6 +18,10 @@ Generate and print an S3 pre-signed URL +Can do the same with the following AWS CLI command: + +aws s3 presign s3:/// [--expires-in ] + Will generate a pre-signed URL even when the bucket and object key don't exist! (you will get a runtime error when requesting the link that the bucket or object doesn't exist) @@ -49,6 +53,7 @@ def main(): parser.add_argument('expiration', nargs='?', default=3600, help='Expiration of URL in seconds') args = parser.parse_args() + # more useful if doing this programmatically as we can do this on the command line via AWS CLI conn = boto3.client('s3') url = conn.generate_presigned_url( 'get_object', From 990e1bb52bf53cab54e2b82d7ca11c6eadf08db5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 11:00:17 +0000 Subject: [PATCH 0639/2779] renamed aws_s3_presign_url.py to aws_s3_presign.py --- aws_s3_presign_url.py => aws_s3_presign.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename aws_s3_presign_url.py => aws_s3_presign.py (100%) diff --git a/aws_s3_presign_url.py b/aws_s3_presign.py similarity index 100% rename from aws_s3_presign_url.py rename to aws_s3_presign.py From dee9ea6ff2a2d9af8c1ca8f6be83e3268e860405 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 16:47:57 +0000 Subject: [PATCH 0640/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index b203357f4..3ab4db6a0 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -509,6 +509,9 @@ dest[137]="aws lambda update-function-code --function-name --zip-file src[138]=' aws elb create-load-balancer --load-balancer-name "$lb_name" ...' dest[138]=' aws elb create-load-balancer --load-balancer-name ...' +src[139]=' in column "blah" of table "blah2"' +dest[139]=' in column "" of table ""' + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 91ba1923014a7d235fa05aba0b97f68acac52657 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 16:48:22 +0000 Subject: [PATCH 0641/2779] updated anonymize.py --- anonymize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 8210af2d6..06e6fa9c9 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.6' +__version__ = '0.10.7' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -317,6 +317,7 @@ def __init__(self): .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), + 'db4': r'(\s(?:in|of)\s+(column|table|database|schema)\s+[\'"])[^\'"]+', 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex), 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, @@ -451,6 +452,7 @@ def __init__(self): 'db': r'\1', 'db2': r'\1', 'db3': r'\1', + 'db4': r'\1<\2>', 'generic': r'\1://', 'generic2': r'\1', 'generic3': r'\1', From 9269ad9e12d8f331b1bf255bdadd8cf32e66bb82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 16:52:33 +0000 Subject: [PATCH 0642/2779] added --ignore-errors --- hive_tables_row_counts.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 80172c21a..39ca80814 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -51,7 +51,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.3.0' +__version__ = '0.4.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -106,6 +106,10 @@ def parse_args(): parser.add_argument('-n', '--krb5-service-name', default=default_service_name, help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + # ignore tables that fail with errors like: + # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' + # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' + parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() @@ -176,8 +180,11 @@ def main(): continue try: get_row_counts(conn, args, database, table, partition_regex) - except impala.error.OperationalError as _: - log.error(_) + except Exception as _: + if args.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise def get_row_counts(conn, args, database, table, partition_regex): log.info("getting partitions for database '%s' table '%s'", database, table) From c997a0b2788c7f5943d263d6977d0e05ea95198d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:09:38 +0000 Subject: [PATCH 0643/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 39ca80814..58122be0d 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -106,9 +106,16 @@ def parse_args(): parser.add_argument('-n', '--krb5-service-name', default=default_service_name, help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') - # ignore tables that fail with errors like: - # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' - # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' +# +# ignore tables that fail with errors like this for Hive (on CDH so MR, no tez): +# +# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long +# +# or this for Impala: +# +# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' +# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' +# parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() From bbf4e0e18d7795b1402a41b267097841b745825d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:11:47 +0000 Subject: [PATCH 0644/2779] added --ignore-errors --- hive_foreach_table.py | 50 ++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index e52eca238..afe850bb2 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -68,7 +68,7 @@ from impala.dbapi import connect __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' logging.basicConfig() log = logging.getLogger(os.path.basename(sys.argv[0])) @@ -122,6 +122,17 @@ def parse_args(): parser.add_argument('-n', '--krb5-service-name', default=default_service_name, help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') +# +# ignore tables that fail with errors like this for Hive (on CDH so MR, no tez): +# +# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long +# +# or this for Impala: +# +# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' +# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' +# + parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() @@ -195,20 +206,29 @@ def main(): if _ == 'db': query = args.query.format(table=table) try: - log.info("running %s", query) - with conn.cursor() as query_cursor: - # doesn't support parameterized query quoting from dbapi spec - query_cursor.execute(query) - for result in query_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) - except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - log.error(_) - except impala.error.ProgrammingError as _: - log.error(_) - # COMPUTE STATS returns no results - if 'Trying to fetch results on an operation with no results' not in str(_): - raise + execute(conn, database, table, query) + except Exception as _: + if args.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise + +def execute(conn, database, table, query): + try: + log.info("running %s", query) + with conn.cursor() as query_cursor: + # doesn't support parameterized query quoting from dbapi spec + query_cursor.execute(query) + for result in query_cursor: + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) + #except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + # log.error(_) + except impala.error.ProgrammingError as _: + log.error(_) + # COMPUTE STATS returns no results + if 'Trying to fetch results on an operation with no results' not in str(_): + raise if __name__ == '__main__': From 4ef8d1b24a614dc27c38f36298c8d5a64f366cc2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:14:50 +0000 Subject: [PATCH 0645/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index afe850bb2..3e2163be3 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -215,7 +215,7 @@ def main(): def execute(conn, database, table, query): try: - log.info("running %s", query) + log.info(" %s.%s - running %s", database, table, query) with conn.cursor() as query_cursor: # doesn't support parameterized query quoting from dbapi spec query_cursor.execute(query) From 5ac651264dc633cf6f96f493e9e9cc24001ccc2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:29:53 +0000 Subject: [PATCH 0646/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 58122be0d..537afffc8 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -107,11 +107,13 @@ def parse_args(): help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') # -# ignore tables that fail with errors like this for Hive (on CDH so MR, no tez): +# ignore tables that fail with errors like: +# +# Hive (CDH has MR, no tez): # # impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long # -# or this for Impala: +# Impala: # # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' From 7a88ad935bd3df2dda29055c091624c8c2eb41f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Jan 2020 17:30:46 +0000 Subject: [PATCH 0647/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 3e2163be3..90b294709 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -123,11 +123,13 @@ def parse_args(): help='Service principal (default: {})'.format(default_service_name)) parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') # -# ignore tables that fail with errors like this for Hive (on CDH so MR, no tez): +# ignore tables that fail with errors like: +# +# Hive (CDH has MR, no tez): # # impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long # -# or this for Impala: +# Impala: # # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' From 9a93bcfd3c376532f40c481d274dc2eb769b0eb4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:09:53 +0000 Subject: [PATCH 0648/2779] updated opentsdb_list_metrics.sh --- opentsdb_list_metrics.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opentsdb_list_metrics.sh b/opentsdb_list_metrics.sh index 3a2675738..6365ffc35 100755 --- a/opentsdb_list_metrics.sh +++ b/opentsdb_list_metrics.sh @@ -72,7 +72,7 @@ done check_bin(){ local bin="$1" - if ! type -P $bin &>/dev/null; then + if ! type -P "$bin" &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi @@ -83,6 +83,8 @@ check_bin jq if [ -z "${DEBUG:-}" ]; then curl_options="$curl_options -s" fi +# split opts +# shellcheck disable=SC2086 curl $curl_options "$tsd_url/api/suggest?type=$metrics&q=&max=2000000000" | jq '.[]' | sed 's/"//g' | From 9e1e67adfa395e8c09c9de2ca67751e1022581aa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:11:43 +0000 Subject: [PATCH 0649/2779] updated test_validate_avro.sh --- tests/test_validate_avro.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_validate_avro.sh b/tests/test_validate_avro.sh index 7308e1ab5..77a11f688 100755 --- a/tests/test_validate_avro.sh +++ b/tests/test_validate_avro.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_avro.py" @@ -26,8 +27,8 @@ section "Testing validate_avro.py" export TIMEOUT=3 if [ $# -gt 0 ]; then - echo "validate_avro.py $@" - ./validate_avro.py $@ + echo "validate_avro.py $*" + ./validate_avro.py "$@" echo fi @@ -82,6 +83,7 @@ echo "testing stdin" ./validate_avro.py - < "$data_dir/test.avro" ./validate_avro.py < "$data_dir/test.avro" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 ./validate_avro.py "$data_dir/test.avro" - < "$data_dir/test.avro" echo @@ -93,10 +95,10 @@ check_broken(){ ./validate_avro.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken avro in '$filename', returned exit code $exitcode" echo - #elif [ $exitcode != 0 ]; then + #elif [ "$exitcode" != 0 ]; then # echo "returned unexpected non-zero exit code $exitcode for broken avro in '$filename'" # exit 1 else From f0cc01acf341e9605afb141496d0f01727dc4037 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:52:10 +0000 Subject: [PATCH 0650/2779] updated test_validate_csv.sh --- tests/test_validate_csv.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_validate_csv.sh b/tests/test_validate_csv.sh index 18822a584..ae415edc1 100755 --- a/tests/test_validate_csv.sh +++ b/tests/test_validate_csv.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_csv.py" @@ -26,8 +27,8 @@ section "Testing validate_csv.py" export TIMEOUT=3 if [ $# -gt 0 ]; then - echo "validate_csv.py $@" - ./validate_csv.py $@ + echo "validate_csv.py $*" + ./validate_csv.py "$@" echo fi @@ -70,6 +71,7 @@ echo "testing stdin" ./validate_csv.py - < "$data_dir/test.csv" ./validate_csv.py < "$data_dir/test.csv" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 ./validate_csv.py "$data_dir/test.csv" - < "$data_dir/test.csv" echo @@ -88,7 +90,7 @@ check_broken(){ ./validate_csv.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken csv in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then @@ -123,7 +125,7 @@ echo "checking blank content is invalid via stdin" check_broken - 2 < "$broken_dir/blank.csv" echo "checking blank content is invalid via stdin piped from /dev/null" -cat /dev/null | check_broken - 2 +check_broken - 2 < /dev/null echo rm -fr "$broken_dir" From 5252e5906bec745975cb2bff08c4bbf286526b17 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:53:49 +0000 Subject: [PATCH 0651/2779] updated test_validate_ini.sh --- tests/test_validate_ini.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh index 32a658e9f..d138aa22b 100755 --- a/tests/test_validate_ini.sh +++ b/tests/test_validate_ini.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_ini.py" @@ -26,8 +27,8 @@ section "Testing validate_ini.py" export TIMEOUT=3 if [ $# -gt 0 ]; then - echo "validate_ini.py $@" - ./validate_ini.py $@ + echo "validate_ini.py $*" + ./validate_ini.py "$@" echo fi @@ -49,7 +50,8 @@ run_fail2(){ run_fail "${@/validate_ini/validate_ini2}" # ignore_run_unqualified } -if [ -f /etc/sssd/sssd.conf -a -r /etc/sssd/sssd.conf ]; then +if [ -f /etc/sssd/sssd.conf ] && + [ -r /etc/sssd/sssd.conf ]; then run ./validate_ini.py /etc/sssd/sssd.conf fi @@ -86,6 +88,7 @@ echo "testing stdin" run2 ./validate_ini.py - < "$data_dir/test.ini" run2 ./validate_ini.py < "$data_dir/test.ini" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 run2 ./validate_ini.py "$data_dir/test.ini" - < "$data_dir/test.ini" echo @@ -115,7 +118,7 @@ check_broken(){ ./validate_ini.py $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken ini in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then @@ -268,10 +271,10 @@ echo hr2 echo "checking blank content is invalid via stdin piped from /dev/null" -cat /dev/null | check_broken - 2 -cat /dev/null | run_fail 2 ./validate_ini.py +check_broken - 2 < /dev/null +run_fail 2 ./validate_ini.py < /dev/null echo "validate_ini2.py blank content is valid:" -cat /dev/null | run ./validate_ini2.py +run ./validate_ini2.py < /dev/null echo hr2 From 5226ea26367dfac04e2023c3fc0c8a61961bde39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:56:56 +0000 Subject: [PATCH 0652/2779] updated test_validate_json.sh --- tests/test_validate_json.sh | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/test_validate_json.sh b/tests/test_validate_json.sh index fb600bae2..279c7b5b9 100755 --- a/tests/test_validate_json.sh +++ b/tests/test_validate_json.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_json.py" @@ -26,8 +27,8 @@ section "Testing validate_json.py" export TIMEOUT=${TIMEOUT:-3} if [ $# -gt 0 ]; then - echo "validate_json.py $@" - ./validate_json.py $@ + echo "validate_json.py $*" + ./validate_json.py "$@" echo fi @@ -68,6 +69,7 @@ echo "testing stdin" ./validate_json.py - < "$data_dir/test.json" ./validate_json.py < "$data_dir/test.json" echo "testing stdin and file mix" +# shellcheck disable=SC2094 ./validate_json.py "$data_dir/test.json" - < "$data_dir/test.json" echo "testing stdin with multirecord" ./validate_json.py -m - < "$data_dir/multirecord.json" @@ -89,7 +91,7 @@ check_broken(){ ./validate_json.py $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken json in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then @@ -210,17 +212,21 @@ echo "checking --permit-single-quotes mode infers multirecord single quoted json echo echo "checking --permit-single-quotes mode works with multirecord single quoted json with mixed quoting (should result in a WARNING message)" -./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" -m 2>&1 | - grep -q WARNING && - echo "Found warning message" || - { echo "failed to raise a WARNING message for mixed quoting"; exit 1; } +if ./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" -m 2>&1 | grep -q WARNING; then + echo "Found warning message" +else + echo "failed to raise a WARNING message for mixed quoting" + exit 1 +fi echo echo "checking --permit-single-quotes mode infers multirecord single quoted json with mixed quoting (should result in a WARNING message)" -./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" 2>&1 | - grep -q WARNING && - echo "Found warning message" || - { echo "failed to raise a WARNING message for mixed quoting"; exit 1; } +if ./validate_json.py -s "$data_dir/multirecord_single_double_mixed_quotes.notjson" 2>&1 | grep -q WARNING; then + echo "Found warning message" +else + echo "failed to raise a WARNING message while inferring mixed quoting" + exit 1 +fi echo # ============================================================================ # @@ -309,7 +315,7 @@ echo "checking blank content is invalid for multirecord via stdin" check_broken - 2 -m < "$broken_dir/blank.json" echo "checking blank content is invalid for multirecord via stdin piped from /dev/null" -cat /dev/null | check_broken - 2 -m +check_broken - 2 -m < /dev/null echo check_broken_sample_files json From f048bf472fabee35d37167116abfe171f33a3b7a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:57:31 +0000 Subject: [PATCH 0653/2779] updated test_validate_ldap_ldif.sh --- tests/test_validate_ldap_ldif.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_ldap_ldif.sh b/tests/test_validate_ldap_ldif.sh index 566d90199..ab5a7654e 100755 --- a/tests/test_validate_ldap_ldif.sh +++ b/tests/test_validate_ldap_ldif.sh @@ -19,13 +19,14 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_ldap_ldif.py" if [ $# -gt 0 ]; then - echo "validate_ldap_ldif.py $@" - ./validate_ldap_ldif.py $@ + echo "validate_ldap_ldif.py $*" + ./validate_ldap_ldif.py "$@" echo fi @@ -60,6 +61,7 @@ echo echo "testing stdin" ./validate_ldap_ldif.py - < "$data_dir/add_ou.ldif" ./validate_ldap_ldif.py < "$data_dir/add_ou.ldif" +# shellcheck disable=SC2094 ./validate_ldap_ldif.py "$data_dir/add_ou.ldif" - < "$data_dir/add_ou.ldif" echo @@ -87,7 +89,7 @@ check_broken(){ ./validate_ldap_ldif.py $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken ldif in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From ef527c942b048eecb4895b54fcb68c2bd2f61864 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:58:00 +0000 Subject: [PATCH 0654/2779] updated test_validate_multimedia.sh --- tests/test_validate_multimedia.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index 1efa5b4e6..9b5565419 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_multimedia.py" @@ -38,8 +39,8 @@ if ! type -P ffmpeg &>/dev/null; then fi if [ $# -gt 0 ]; then - echo "validate_multimedia.py $@" - ./validate_multimedia.py $@ + echo "validate_multimedia.py $*" + ./validate_multimedia.py "$@" echo fi @@ -87,7 +88,7 @@ check_broken(){ ./validate_multimedia.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken media in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From a36698204f21cc34bdaac7a21f5112445be04175 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:58:30 +0000 Subject: [PATCH 0655/2779] updated test_validate_parquet.sh --- tests/test_validate_parquet.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_parquet.sh b/tests/test_validate_parquet.sh index 01d8a0e0f..eb6e78e8b 100755 --- a/tests/test_validate_parquet.sh +++ b/tests/test_validate_parquet.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_parquet.py" @@ -32,8 +33,8 @@ fi #export TIMEOUT=10 if [ $# -gt 0 ]; then - echo "validate_parquet.py $@" - ./validate_parquet.py $@ + echo "validate_parquet.py $*" + ./validate_parquet.py "$@" echo fi @@ -84,6 +85,7 @@ echo "testing stdin" ./validate_parquet.py - < "$data_dir/test.parquet" ./validate_parquet.py < "$data_dir/test.parquet" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 ./validate_parquet.py "$data_dir/test.parquet" - < "$data_dir/test.parquet" echo @@ -95,7 +97,7 @@ check_broken(){ ./validate_parquet.py -t 5 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken parquet in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From 8b4d4951ead31e3d0fe4000a1b763ce0d30f0897 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:59:09 +0000 Subject: [PATCH 0656/2779] updated test_validate_xml.sh --- tests/test_validate_xml.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_xml.sh b/tests/test_validate_xml.sh index ae6479497..5742faf45 100755 --- a/tests/test_validate_xml.sh +++ b/tests/test_validate_xml.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_xml.py" @@ -26,8 +27,8 @@ section "Testing validate_xml.py" export TIMEOUT=3 if [ $# -gt 0 ]; then - echo "validate_xml.py $@" - ./validate_xml.py $@ + echo "validate_xml.py $*" + ./validate_xml.py "$@" echo fi @@ -60,6 +61,7 @@ echo echo "testing stdin" ./validate_xml.py - < "$data_dir/simple.xml" ./validate_xml.py < "$data_dir/simple.xml" +# shellcheck disable=SC2094 ./validate_xml.py "$data_dir/simple.xml" - < "$data_dir/simple.xml" echo @@ -77,7 +79,7 @@ check_broken(){ ./validate_xml.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken xml in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From 6d51e104f939ceb2dcb70907583cba587b15c167 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 14:59:46 +0000 Subject: [PATCH 0657/2779] updated test_validate_yaml.sh --- tests/test_validate_yaml.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_validate_yaml.sh b/tests/test_validate_yaml.sh index 7ab5b2975..d8b0e265c 100755 --- a/tests/test_validate_yaml.sh +++ b/tests/test_validate_yaml.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing validate_yaml.py" @@ -29,8 +30,8 @@ if is_inside_docker; then fi if [ $# -gt 0 ]; then - echo "validate_yaml.py $@" - ./validate_yaml.py $@ + echo "validate_yaml.py $*" + ./validate_yaml.py "$@" echo fi @@ -65,6 +66,7 @@ echo "testing stdin" ./validate_yaml.py - < "$data_dir/test.yaml" ./validate_yaml.py < "$data_dir/test.yaml" echo "testing stdin mixed with filename" +# shellcheck disable=SC2094 ./validate_yaml.py "$data_dir/test.yaml" - < "$data_dir/test.yaml" echo @@ -82,7 +84,7 @@ check_broken(){ ./validate_yaml.py -t 1 $options "$filename" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ "$exitcode" = "$expected_exitcode" ]; then echo "successfully detected broken yaml in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From f4cdfb53a2e7c5149b45096adfff722b9b55573f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 15:01:33 +0000 Subject: [PATCH 0658/2779] updated opentsdb_list_metrics_hbase.sh --- opentsdb_list_metrics_hbase.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opentsdb_list_metrics_hbase.sh b/opentsdb_list_metrics_hbase.sh index 56aaddfa9..945b108ff 100755 --- a/opentsdb_list_metrics_hbase.sh +++ b/opentsdb_list_metrics_hbase.sh @@ -73,7 +73,7 @@ done check_bin(){ local bin="$1" - if ! type -P $bin &>/dev/null; then + if ! type -P "$bin" &>/dev/null; then echo "'$bin' command not found in \$PATH ($PATH)" exit 1 fi @@ -106,6 +106,7 @@ for line in sys.stdin: # ts = metrics[metric] print("{}\t{}\t{}".format(ts, time.strftime("%F %T", time.localtime(int(ts)/1000)), metric)) EOF + # shellcheck disable=SC2064 trap "rm '$tmp_python_script'" EXIT hbase shell <<< "scan 'tsdb-uid', { COLUMNS => 'name:$metrics', VERSIONS => 1 }" 2>/dev/null | From 771824123ff5cb0fa6cad022ecff7067c92ef0bf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 16 Jan 2020 17:58:09 +0000 Subject: [PATCH 0659/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 537afffc8..d4d685f26 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -118,7 +118,7 @@ def parse_args(): # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' # - parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') + parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') args = parser.parse_args() @@ -190,6 +190,8 @@ def main(): try: get_row_counts(conn, args, database, table, partition_regex) except Exception as _: + # invalid query handle and similar errors happen at higher level + # as they are not query specific, will not be caught here so still error out if args.ignore_errors: log.error("database '%s' table '%s': %s", database, table, _) continue From 010dfb90db60aef2c9984a8afd572a08e1cb82af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 11:29:12 +0000 Subject: [PATCH 0660/2779] updated all.sh --- tests/all.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/all.sh b/tests/all.sh index fabd97170..18dddfc25 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -17,6 +17,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# shellcheck disable=SC1090 . "$srcdir/utils.sh" # imported by utils.sh above From cdbf33b67cd8f792f070d9a453f11ea9e6c4728f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 11:29:36 +0000 Subject: [PATCH 0661/2779] updated check.sh --- tests/check.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/check.sh b/tests/check.sh index 4541f684b..07e1bd9c2 100755 --- a/tests/check.sh +++ b/tests/check.sh @@ -21,12 +21,12 @@ check(){ msg=$2 echo hr2 - echo $msg + echo "$msg" hr2 echo - echo cmd: $cmd + echo "cmd: $cmd" echo - if eval $cmd; then + if eval "$cmd"; then echo echo "SUCCESS" else From 01671a19207886fe96a42c35146293488e67a79c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 11:46:55 +0000 Subject: [PATCH 0662/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 3ab4db6a0..829cf04b0 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -18,7 +18,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -test_nums="${@:-}" +test_nums="${*:-}" parallel="" if [ "$test_nums" = "p" ]; then parallel="1" @@ -27,6 +27,7 @@ fi cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Anonymize" @@ -45,7 +46,7 @@ if [ -z "$test_nums" ]; then echo echo "checking file args:" run++ - if [ `$anonymize -ae README.md | wc -l` -gt 100 ]; then + if [ "$($anonymize -ae README.md | wc -l)" -gt 100 ]; then echo "SUCCEEDED - anonymized README.md > 100 lines" else echo "FAILED - suspicious README.md file arg result came to <= 100 lines" @@ -506,6 +507,7 @@ dest[136]="arn:aws:acm:us-east-1::function::7" src[137]="aws lambda update-function-code --function-name hari-test --zip-file fileb://myfunction.zip" dest[137]="aws lambda update-function-code --function-name --zip-file fileb://" +# shellcheck disable=SC2016 src[138]=' aws elb create-load-balancer --load-balancer-name "$lb_name" ...' dest[138]=' aws elb create-load-balancer --load-balancer-name ...' @@ -523,12 +525,16 @@ dest[139]=' in column "" of table "
"' args="-aPe" test_anonymize(){ run++ - src="$1" - dest="$2" + # shellcheck disable=SC2178 + local src="$1" + # shellcheck disable=SC2178 + local dest="$2" #[ -z "${src[$i]:-}" ] && { echo "skipping test $i..."; continue; } # didn't work for \e escape codes for ANSI stripping test #result="$(echo -e "$src" | $anonymize $args)" + # shellcheck disable=SC2128 result="$($anonymize $args <<< "$src")" + # shellcheck disable=SC2128 if grep -xFq -- "$dest" <<< "$result"; then echo -n "SUCCEEDED anonymization test $i" if [ -n "${SHOW_OUTPUT:-}" ]; then @@ -560,7 +566,7 @@ fi # this gives the number of elements and prevents testing the last element(s) if commenting something out in the middle #for (( i = 0 ; i < ${#src[@]} ; i++ )); do run_tests(){ - test_numbers="${@:-${!src[@]}}" + test_numbers="${*:-${!src[*]}}" for i in $test_numbers; do [ -n "${src[$i]:-}" ] || { echo "code error: src[$i] not defined"; exit 1; } [ -n "${dest[$i]:-}" ] || { echo "code error: dest[$i] not defined"; exit 1; } @@ -614,6 +620,8 @@ if [ -n "$parallel" ]; then fi echo +# run_count assigned in utils lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" time_taken "$start_time" "SUCCESS! All tests for $anonymize completed in" echo From a57afb0d1591baa0a030b206b54bbcde9761ecdb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 14:09:11 +0000 Subject: [PATCH 0663/2779] updated python3.sh --- tests/python3.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python3.sh b/tests/python3.sh index eeaf5f9f3..9cd102ac1 100755 --- a/tests/python3.sh +++ b/tests/python3.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh pip install caniusepython3 From 006cc4e89ccbdd188d6b96e5b1135a5f62ab16a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 14:11:05 +0000 Subject: [PATCH 0664/2779] updated syntax.sh --- tests/syntax.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/syntax.sh b/tests/syntax.sh index 26a8f16c8..ea6b33a6a 100755 --- a/tests/syntax.sh +++ b/tests/syntax.sh @@ -19,20 +19,21 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh -for x in $(echo *.py *.jy 2>/dev/null); do - isExcluded "$x" && continue +while read -r prog; do + isExcluded "$prog" && continue if type -P flake8 &> /dev/null; then - echo "flake8 $x" - flake8 --max-line-length=120 --statistics $x + echo "flake8 $prog" + flake8 --max-line-length=120 --statistics "$prog" echo; hr; echo fi for y in pyflakes pychecker; do - if type -P $y &>/dev/null; then - echo "$y $x" - $y $x + if type -P "$y" &>/dev/null; then + echo "$y $prog" + "$y" "$prog" echo; hr; echo fi done -done +done < <(find . -type f -name '*.py' -o -type f -name '*.jy') From b8f6d77b752dbb718c2eb4ebde40cf66c5cc7171 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Jan 2020 15:18:49 +0000 Subject: [PATCH 0665/2779] updated test_apache-drill.sh --- tests/test_apache-drill.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_apache-drill.sh b/tests/test_apache-drill.sh index 03c6539a8..c9a675207 100755 --- a/tests/test_apache-drill.sh +++ b/tests/test_apache-drill.sh @@ -20,11 +20,12 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1090 . "$srcdir/utils.sh" section "A p a c h e D r i l l" -export APACHE_DRILL_VERSIONS="${@:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 1.14 1.15 latest}}" +export APACHE_DRILL_VERSIONS="${*:-${APACHE_DRILL_VERSIONS:-0.7 0.8 0.9 1.0 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 1.12 1.13 1.14 1.15 latest}}" APACHE_DRILL_HOST="${DOCKER_HOST:-${APACHE_DRILL_HOST:-${HOST:-localhost}}}" APACHE_DRILL_HOST="${APACHE_DRILL_HOST##*/}" @@ -45,6 +46,7 @@ test_apache_drill(){ echo "getting Apache Drill dynamic port mappings:" docker_compose_port "Apache Drill" hr + # shellcheck disable=SC2153 when_ports_available "$APACHE_DRILL_HOST" "$APACHE_DRILL_PORT" hr when_url_content "http://$APACHE_DRILL_HOST:$APACHE_DRILL_PORT/status" "Running" @@ -60,15 +62,19 @@ test_apache_drill(){ hr APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_apache_drill.py $non_drill_node1 $non_drill_node2 + # shellcheck disable=SC2097,SC2098 APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" run_grep "^$APACHE_DRILL_HOST:$APACHE_DRILL_PORT$" ./find_active_apache_drill.py $non_drill_node1 "$APACHE_DRILL_HOST:$APACHE_DRILL_PORT" # Drill 1.7+ only - if [ "$version" = "latest" ] || [[ "$version" > 1.6 ]]; then + if [ "$version" = "latest" ] || [ "$(bc <<< "$version > 1.6")" = 1 ]; then APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_apache_drill2.py $non_drill_node1 $non_drill_node2 + # shellcheck disable=SC2097,SC2098 APACHE_DRILL_PORT="$APACHE_DRILL_PORT_DEFAULT" run_grep "^$APACHE_DRILL_HOST:$APACHE_DRILL_PORT$" ./find_active_apache_drill2.py $non_drill_node1 "$APACHE_DRILL_HOST:$APACHE_DRILL_PORT" fi + # run_count defined in util lib + # shellcheck disable=SC2154 echo "Completed $run_count Apache Drill tests" hr [ -n "${KEEPDOCKER:-}" ] || From 6d417263c0d2ac21e50759915489ca47c6aa2e84 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 21 Jan 2020 14:54:43 +0000 Subject: [PATCH 0666/2779] moved STS token to evaluate before secret key --- anonymize.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/anonymize.py b/anonymize.py index 06e6fa9c9..07bc730ac 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.7' +__version__ = '0.10.8' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -300,8 +300,8 @@ def __init__(self): 'aws2': r'\b(arn:[^:]+:[^:]+:[^:]*:)\d*:[\w/.-]+', # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html 'aws3': r'\bAKIA[A-Za-z0-9]{16}\b', # access key - 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key - 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix + 'aws4': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{238,}', # STS token - no \b at end as it'll stop before '==' suffix + 'aws5': r'\b[A-Za-z0-9][A-Za-z0-9/+=-]{38}[A-Za-z0-9]\b', # secret key 'aws6': r'\bASIA[A-Za-z0-9]{16}\b', # sts temporary access key 'aws7': r'\bsg-[A-Za-z0-9]{8}(?\2<\3>', 'aws2': r'\1:', 'aws3': r'', - 'aws4': r'', - 'aws5': r'', + 'aws4': r'', + 'aws5': r'', 'aws6': r'', 'aws7': r'', 'aws8': r'\1:///', From 3574d27cf57070c79bbd617b48f0b0312a5e50d2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:15:37 +0000 Subject: [PATCH 0667/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 384e5f8b3..a134b3ec8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 384e5f8b366fdeb6a73363dbf388824f5de5e7bc +Subproject commit a134b3ec87c26561ee43180be67b662665a72542 From 70395a5e784481b03c185e99009215b9f699bbb9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:15:38 +0000 Subject: [PATCH 0668/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f3fb9b2a9..9cf83f21c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f3fb9b2a9dedb8a4ed7e85183f1739ad2a63fd92 +Subproject commit 9cf83f21c39fdb250d4e8a155d08220b204adaee From 48ec6ca7d745d639253b9f9059dfbde1fd7965a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:44:23 +0000 Subject: [PATCH 0669/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a134b3ec8..1e33eaec4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a134b3ec87c26561ee43180be67b662665a72542 +Subproject commit 1e33eaec4117818ee32e3644e101a715f5a66de0 From 8b8411cc017d44bd3acb0708d41657a16691fdaa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:44:24 +0000 Subject: [PATCH 0670/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9cf83f21c..4dcc62e62 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9cf83f21c39fdb250d4e8a155d08220b204adaee +Subproject commit 4dcc62e62cbc569a0422a1f57be20ceef32054d0 From d5ca483db87946ac187209d168965c2d541a5988 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:44:42 +0000 Subject: [PATCH 0671/2779] updated README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b7f45f612..cc620fcad 100644 --- a/README.md +++ b/README.md @@ -336,12 +336,13 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (Hive, Impala, MySQL, PostgreSQL, Cassandra CQL, Apache Drill, Couchbase N1QL, Microsoft SQL Server, Oracle, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... -* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Hadoop, Docker, Kafka, Elasticsearch, RabbitMQ, Redis, HBase, Solr, Cassandra, ZooKeeper, HDFS, Yarn, Hive, Presto, Drill, Impala, Consul, Spark, Jenkins, Travis CI, Git, MySQL, Linux, DNS, Whois, SSL Certs, Yum Security Updates, Kubernetes, Mesos, Riak, MongoDB, Memcached, Couchbase, CouchDB, Neo4j, Ambari, Cloudera, Hortonworks, MapR etc. +* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) * [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 100+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies +* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... + * [HAProxy-configs](https://github.com/harisekhon/haproxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. * [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 From e3edbb228461ea458a9eafb07b35ba5b59ca51e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 10:58:34 +0000 Subject: [PATCH 0672/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index cc620fcad..114ee40ca 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Hari Sekhon Cloud & Big Data Contractor, United Kingdom https://www.linkedin.com/in/harisekhon +###### (you're welcome to connect with me on LinkedIn) ##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### From 6822bed726e7bbbfa251e980496c107482e0454b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 11:51:32 +0000 Subject: [PATCH 0673/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 114ee40ca..c32604247 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Hari Sekhon Cloud & Big Data Contractor, United Kingdom -https://www.linkedin.com/in/harisekhon +[https://www.linkedin.com/in/harisekhon](https://www.linkedin.com/in/harisekhon) ###### (you're welcome to connect with me on LinkedIn) ##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### From 112337e86d8bcde11bf63f5735322ecf6aecc82a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:20:10 +0000 Subject: [PATCH 0674/2779] updated test_validate_avro.sh --- tests/test_validate_avro.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_avro.sh b/tests/test_validate_avro.sh index 77a11f688..e7c595d51 100755 --- a/tests/test_validate_avro.sh +++ b/tests/test_validate_avro.sh @@ -90,7 +90,7 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e ./validate_avro.py -t 1 $options "$filename" exitcode=$? From b2777b7efbfd0a014c04d181c1e328e8c5dac63e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:20:25 +0000 Subject: [PATCH 0675/2779] updated test_validate_csv.sh --- tests/test_validate_csv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_csv.sh b/tests/test_validate_csv.sh index ae415edc1..ac9813a03 100755 --- a/tests/test_validate_csv.sh +++ b/tests/test_validate_csv.sh @@ -85,7 +85,7 @@ hr2 check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e ./validate_csv.py -t 1 $options "$filename" exitcode=$? From 96f3e864c7f2bbef8ff68a32bb95627179e938e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:20:43 +0000 Subject: [PATCH 0676/2779] updated test_validate_ini.sh --- tests/test_validate_ini.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh index d138aa22b..bcfaebfd8 100755 --- a/tests/test_validate_ini.sh +++ b/tests/test_validate_ini.sh @@ -113,7 +113,7 @@ export TIMEOUT=1 check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e ./validate_ini.py $options "$filename" exitcode=$? From d2bcdba4378d65cd583b59034640b967ebc540e2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:21:21 +0000 Subject: [PATCH 0677/2779] updated test_validate_json.sh --- tests/test_validate_json.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_json.sh b/tests/test_validate_json.sh index 279c7b5b9..39b24306b 100755 --- a/tests/test_validate_json.sh +++ b/tests/test_validate_json.sh @@ -86,8 +86,9 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_json.py $options "$filename" exitcode=$? set -e From 89149c79ea6ff6bcbbd64c42334ec069c8d48eb1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:21:46 +0000 Subject: [PATCH 0678/2779] updated test_validate_ldap_ldif.sh --- tests/test_validate_ldap_ldif.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_ldap_ldif.sh b/tests/test_validate_ldap_ldif.sh index ab5a7654e..53af23074 100755 --- a/tests/test_validate_ldap_ldif.sh +++ b/tests/test_validate_ldap_ldif.sh @@ -84,8 +84,9 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_ldap_ldif.py $options "$filename" exitcode=$? set -e From 494164e76d0681e447eb211567183316b1550cd0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:22:07 +0000 Subject: [PATCH 0679/2779] updated test_validate_multimedia.sh --- tests/test_validate_multimedia.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index 9b5565419..7302ca55d 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -83,8 +83,9 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_multimedia.py -t 1 $options "$filename" exitcode=$? set -e From 8a843376d8632c0818c91b49185c5d8b5aa66abc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:22:35 +0000 Subject: [PATCH 0680/2779] updated test_validate_parquet.sh --- tests/test_validate_parquet.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_parquet.sh b/tests/test_validate_parquet.sh index eb6e78e8b..d75942e84 100755 --- a/tests/test_validate_parquet.sh +++ b/tests/test_validate_parquet.sh @@ -92,8 +92,9 @@ echo check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_parquet.py -t 5 $options "$filename" exitcode=$? set -e From b64eba0958f2f966c2e9daa92200a703798ec430 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:23:04 +0000 Subject: [PATCH 0681/2779] updated test_validate_xml.sh --- tests/test_validate_xml.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_xml.sh b/tests/test_validate_xml.sh index 5742faf45..37b1ee106 100755 --- a/tests/test_validate_xml.sh +++ b/tests/test_validate_xml.sh @@ -74,8 +74,9 @@ echo "Now trying non-xml files to detect successful failure:" check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_xml.py -t 1 $options "$filename" exitcode=$? set -e From 5091b540113a208d52428b3d5bbf829cbea6b3ba Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:23:26 +0000 Subject: [PATCH 0682/2779] updated test_validate_yaml.sh --- tests/test_validate_yaml.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_validate_yaml.sh b/tests/test_validate_yaml.sh index d8b0e265c..098cf827d 100755 --- a/tests/test_validate_yaml.sh +++ b/tests/test_validate_yaml.sh @@ -79,8 +79,9 @@ echo "Now trying non-yaml files to detect successful failure:" check_broken(){ local filename="$1" local expected_exitcode="${2:-2}" - local options="${@:3}" + local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_yaml.py -t 1 $options "$filename" exitcode=$? set -e From a8a0bacce82f8bc0372d8531fc28ab7fb7e81be8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:23:46 +0000 Subject: [PATCH 0683/2779] updated test_validate_avro.sh --- tests/test_validate_avro.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_validate_avro.sh b/tests/test_validate_avro.sh index e7c595d51..d8dab1d88 100755 --- a/tests/test_validate_avro.sh +++ b/tests/test_validate_avro.sh @@ -92,6 +92,7 @@ check_broken(){ local expected_exitcode="${2:-2}" local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_avro.py -t 1 $options "$filename" exitcode=$? set -e From b7e82109e1439b4a2c3c75956f50167d5c321b72 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:24:03 +0000 Subject: [PATCH 0684/2779] updated test_validate_csv.sh --- tests/test_validate_csv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_validate_csv.sh b/tests/test_validate_csv.sh index ac9813a03..680b63390 100755 --- a/tests/test_validate_csv.sh +++ b/tests/test_validate_csv.sh @@ -87,6 +87,7 @@ check_broken(){ local expected_exitcode="${2:-2}" local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_csv.py -t 1 $options "$filename" exitcode=$? set -e From ec37e03857d344b85dd85d8abde8816c2b5ea624 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:24:14 +0000 Subject: [PATCH 0685/2779] updated test_validate_ini.sh --- tests/test_validate_ini.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh index bcfaebfd8..f2282be6a 100755 --- a/tests/test_validate_ini.sh +++ b/tests/test_validate_ini.sh @@ -115,6 +115,7 @@ check_broken(){ local expected_exitcode="${2:-2}" local options="${*:3}" set +e + # shellcheck disable=SC2086 ./validate_ini.py $options "$filename" exitcode=$? set -e From 02ec92023ace876cf5ccd35d88cb9f1ef16d14fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:31:53 +0000 Subject: [PATCH 0686/2779] updated test_timeout.sh --- tests/test_timeout.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_timeout.sh b/tests/test_timeout.sh index cadfde058..4ce6753d2 100755 --- a/tests/test_timeout.sh +++ b/tests/test_timeout.sh @@ -15,7 +15,6 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x - srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -$srcdir/../timeout.py -t 2 sleep 10 || : +"$srcdir/../timeout.py" -t 2 sleep 10 || : From 0917b820ac11d2a96aa1ce9bf23550f45819da48 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:33:19 +0000 Subject: [PATCH 0687/2779] updated test_timeout.sh --- tests/test_timeout.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_timeout.sh b/tests/test_timeout.sh index 4ce6753d2..26d38d589 100755 --- a/tests/test_timeout.sh +++ b/tests/test_timeout.sh @@ -17,4 +17,7 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -"$srcdir/../timeout.py" -t 2 sleep 10 || : +# shellcheck disable=SC1090 +. "$srcdir/../bash-tools/lib/utils.sh" + +run_fail 3 "$srcdir/../timeout.py" -t 2 sleep 10 From b3117a12868df50da8a185de1b4ee542bb0eeb75 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:34:50 +0000 Subject: [PATCH 0688/2779] updated test_strip_ansi_escape_codes.sh --- tests/test_strip_ansi_escape_codes.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_strip_ansi_escape_codes.sh b/tests/test_strip_ansi_escape_codes.sh index b885c1878..d0effe274 100755 --- a/tests/test_strip_ansi_escape_codes.sh +++ b/tests/test_strip_ansi_escape_codes.sh @@ -15,6 +15,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Strip ANSI Escape Codes" @@ -23,11 +24,11 @@ name="strip_ansi_escape_codes.py" start_time=$(date +%s) -if is_mac; then - cat_opts="-e" -else - cat_opts="-A" -fi +#if is_mac; then +# cat_opts="-e" +#else +# cat_opts="-A" +#fi run++ if echo "some highlighted content" | grep --color=yes highlighted | @@ -42,6 +43,7 @@ fi hr tmp=$(mktemp /tmp/strip_ansi_escape_codes.XXXXX) +# shellcheck disable=SC2064,SC2086 trap "rm $tmp" $TRAP_SIGNALS echo @@ -62,6 +64,8 @@ tee /dev/stderr | fi echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" time_taken "$start_time" "All version tests for $name completed in" echo From 0c541e32bf4914f6ff9fb22eb1aab28d16bb8a8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:36:19 +0000 Subject: [PATCH 0689/2779] updated test_center.sh --- tests/test_center.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_center.sh b/tests/test_center.sh index 976466e1a..0dfd434f8 100755 --- a/tests/test_center.sh +++ b/tests/test_center.sh @@ -22,6 +22,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing center.py" @@ -127,6 +128,8 @@ echo "testing spacing with stdin:" run_output "$expected" ./center.py -s <<< " " echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Completed $run_count tests" echo echo "All tests for center.py completed successfully" From e8a2007fcc5c8450a8fabb345ae1f038d4530db7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:39:54 +0000 Subject: [PATCH 0690/2779] updated welcome.py --- welcome.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/welcome.py b/welcome.py index 97f6f173f..0d93b836f 100755 --- a/welcome.py +++ b/welcome.py @@ -45,7 +45,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '2.0.1' +__version__ = '2.0.2' class Welcome(CLI): @@ -56,6 +56,7 @@ def __init__(self): # Python 3.x # super().__init__() self.quick = False + self.timeout_default = 20 @staticmethod def case_user(user): From c2118940c23924ebe7d5c12ff7d207fc69523087 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:40:38 +0000 Subject: [PATCH 0691/2779] updated test_docker.sh --- tests/test_docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_docker.sh b/tests/test_docker.sh index ddb02baa7..8d00d3be3 100755 --- a/tests/test_docker.sh +++ b/tests/test_docker.sh @@ -19,7 +19,10 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1091 . "bash-tools/lib/docker.sh" + +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Docker Image" From 10024bd385eb8055dec70de3bab07a667dcbd1b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:52:09 +0000 Subject: [PATCH 0692/2779] updated test_docker_registry_show_tags.sh --- tests/test_docker_registry_show_tags.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_docker_registry_show_tags.sh b/tests/test_docker_registry_show_tags.sh index d3d72b932..bc0612399 100755 --- a/tests/test_docker_registry_show_tags.sh +++ b/tests/test_docker_registry_show_tags.sh @@ -19,8 +19,10 @@ srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir2/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" srcdir="$srcdir2" @@ -75,12 +77,13 @@ private_key="registry.key" certificate="registry.crt" htpasswd="registry.htpasswd" csr="registry.csr" -if ! [ -f "$private_key" -a -f "$certificate" ]; then +if ! [ -f "$private_key" ] && + [ -f "$certificate" ]; then echo "Generating sample SSL certificates:" echo openssl genrsa -out "$private_key" 2048 echo - yes "" | openssl req -new -key "$private_key" -out "$csr" || : + yes "." | openssl req -new -key "$private_key" -out "$csr" || : echo openssl x509 -req -days 3650 -in "$csr" -signkey "$private_key" -out "$certificate" echo @@ -108,6 +111,7 @@ echo "getting dynamic Docker Registry port mapping:" docker_compose_port "Docker Registry" hr +# shellcheck disable=SC2153 if [ -z "$DOCKER_REGISTRY_PORT" ]; then echo "DOCKER_REGISTRY_PORT not found from running container, did container fail to start up properly?" exit 1 From b2347e4e2eb4df7862717122ffcaa4e7e2f5fc4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:53:12 +0000 Subject: [PATCH 0693/2779] updated test_dockerfiles_check_git_branches.sh --- tests/test_dockerfiles_check_git_branches.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_dockerfiles_check_git_branches.sh b/tests/test_dockerfiles_check_git_branches.sh index ec2be2a11..2529a2c9c 100755 --- a/tests/test_dockerfiles_check_git_branches.sh +++ b/tests/test_dockerfiles_check_git_branches.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Dockerfiles Check Git branches" From 9f8b74267ee327995b204fe5f4d83ae9c2b9c442 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:53:22 +0000 Subject: [PATCH 0694/2779] updated test_dockerfiles_check_git_tags.sh --- tests/test_dockerfiles_check_git_tags.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_dockerfiles_check_git_tags.sh b/tests/test_dockerfiles_check_git_tags.sh index 33556738c..ada872f3c 100755 --- a/tests/test_dockerfiles_check_git_tags.sh +++ b/tests/test_dockerfiles_check_git_tags.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Dockerfiles Check Git Tags" From 8fb94d8288a7f4f45d62a8cb38a3a86342e4e29c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:54:05 +0000 Subject: [PATCH 0695/2779] updated test_dockerhub_search.sh --- tests/test_dockerhub_search.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index 73b8bf7b5..ee82c5bb8 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing DockerHub Show Tags" @@ -33,6 +35,7 @@ check './dockerhub_search.py harisekhon -n 30' "DockerHub Search for harisekhon check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED +# shellcheck disable=SC2016 check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" echo From b0e31398e31366d2d59042d5a31a4547faa35c5b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:54:27 +0000 Subject: [PATCH 0696/2779] updated test_dockerhub_show_tags.sh --- tests/test_dockerhub_show_tags.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_dockerhub_show_tags.sh b/tests/test_dockerhub_show_tags.sh index 71f424a3f..8d052f174 100755 --- a/tests/test_dockerhub_show_tags.sh +++ b/tests/test_dockerhub_show_tags.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing DockerHub Show Tags" @@ -39,6 +41,8 @@ echo echo echo "All DockerHub Show Tags tests completed successfully" echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" time_taken "$start_time" "DockerHub Show Tags tests completed in" echo From a4f4d50064a2a4e4a449e5cfa5be7eb967f6efb4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:55:55 +0000 Subject: [PATCH 0697/2779] updated test_elasticsearch.sh --- tests/test_elasticsearch.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_elasticsearch.sh b/tests/test_elasticsearch.sh index e18fd8d09..5abd6e53f 100755 --- a/tests/test_elasticsearch.sh +++ b/tests/test_elasticsearch.sh @@ -19,11 +19,12 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "E l a s t i c s e a r c h" -export ELASTICSEARCH_VERSIONS="${@:-${ELASTICSEARCH_VERSIONS:-latest 1.3 1.4 1.5 1.6 1.7 2.0 2.1 2.2 2.3 2.4 5.0 5.1 5.2 5.3 5.4 5.5 5.6 6.0.1 6.1.1}}" +export ELASTICSEARCH_VERSIONS="${*:-${ELASTICSEARCH_VERSIONS:-latest 1.3 1.4 1.5 1.6 1.7 2.0 2.1 2.2 2.3 2.4 5.0 5.1 5.2 5.3 5.4 5.5 5.6 6.0.1 6.1.1}}" ELASTICSEARCH_HOST="${DOCKER_HOST:-${ELASTICSEARCH_HOST:-${HOST:-localhost}}}" ELASTICSEARCH_HOST="${ELASTICSEARCH_HOST##*/}" @@ -43,7 +44,7 @@ test_elasticsearch(){ local version="$1" section2 "Setting up Elasticsearch $version test container" if [ "$version" != "latest" ] && [ "${version:0:1}" -ge 6 ]; then - local export COMPOSE_FILE="$srcdir/docker/$DOCKER_SERVICE-elastic.co-docker-compose.yml" + export COMPOSE_FILE="$srcdir/docker/$DOCKER_SERVICE-elastic.co-docker-compose.yml" fi docker_compose_pull VERSION="$version" docker-compose up -d @@ -63,6 +64,7 @@ test_elasticsearch(){ ELASTICSEARCH_PORT="$ELASTICSEARCH_PORT_DEFAULT" \ ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_elasticsearch.py $non_es_node1 $non_es_node2 + # shellcheck disable=SC2097,SC2098 ELASTICSEARCH_PORT="$ELASTICSEARCH_PORT_DEFAULT" \ run_grep "^$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT$" ./find_active_elasticsearch.py $non_es_node1 $non_es_node2 "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT" From 0803c70901d57707ae36b71d3b0cab1d5764f3dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:58:00 +0000 Subject: [PATCH 0698/2779] updated test_find_active_server.sh --- tests/test_find_active_server.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_find_active_server.sh b/tests/test_find_active_server.sh index 74bb8e494..1bde9e0e6 100755 --- a/tests/test_find_active_server.sh +++ b/tests/test_find_active_server.sh @@ -20,6 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1091 . bash-tools/lib/utils.sh set +e +o pipefail @@ -174,19 +175,19 @@ count_socket_attempts=0 found_google_socket=0 found_duckduckgo_socket=0 run++ -for x in {1..10}; do +for _ in {1..10}; do echo -n . - let count_socket_attempts+=1 + ((count_socket_attempts+=1)) output="$(./find_active_server.py -n1 --random --port 80 $WEBSITE1 $WEBSITE2)" if [ "$output" = "$WEBSITE2" ]; then found_google_socket=1 elif [ "$output" = "$WEBSITE1" ]; then found_duckduckgo_socket=1 fi - [ $found_google_socket -eq 1 -a $found_duckduckgo_socket -eq 1 ] && break + [ $found_google_socket -eq 1 ] && [ $found_duckduckgo_socket -eq 1 ] && break done echo -if [ $found_google_socket -eq 1 -a $found_duckduckgo_socket -eq 1 ]; then +if [ $found_google_socket -eq 1 ] && [ $found_duckduckgo_socket -eq 1 ]; then echo "Found both $WEBSITE1 and $WEBSITE2 in results from $count_socket_attempts --random runs" else die "Failed to return both $WEBSITE1 and $WEBSITE2 in results from $count_socket_attempts --random runs" @@ -210,25 +211,27 @@ count_http_attempts=0 found_google_http=0 found_duckduckgo_http=0 run++ -for x in {1..10}; do +for _ in {1..10}; do echo -n . - let count_http_attempts+=1 + ((count_http_attempts+=1)) output="$(./find_active_server.py -n1 --http --random $WEBSITE1 $WEBSITE2)" if [ "$output" = "$WEBSITE2" ]; then found_google_http=1 elif [ "$output" = "$WEBSITE1" ]; then found_duckduckgo_http=1 fi - [ $found_google_http -eq 1 -a $found_duckduckgo_http -eq 1 ] && break + [ $found_google_http -eq 1 ] && [ $found_duckduckgo_http -eq 1 ] && break done echo -if [ $found_google_http -eq 1 -a $found_duckduckgo_http -eq 1 ]; then +if [ $found_google_http -eq 1 ] && [ $found_duckduckgo_http -eq 1 ]; then echo "Found both $WEBSITE1 and $WEBSITE2 in results from $count_http_attempts --random runs" else die "Failed to return both $WEBSITE1 and $WEBSITE2 in results from $count_http_attempts --random runs" fi hr echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Tests run: $run_count" time_taken "$start_time" "find_active_server.py tests completed in" echo From cc18f5690ac8b7e8d67e24eaaefc3de6c25a08a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 13:59:05 +0000 Subject: [PATCH 0699/2779] updated test_find_duplicate_files.sh --- tests/test_find_duplicate_files.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_find_duplicate_files.sh b/tests/test_find_duplicate_files.sh index 807508f52..9a4e951c9 100755 --- a/tests/test_find_duplicate_files.sh +++ b/tests/test_find_duplicate_files.sh @@ -20,6 +20,7 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "find_duplicate_files.py" @@ -29,6 +30,7 @@ start_time="$(start_timer "find_duplicate_files.py test")" testdir1="$(cd tests/data/ && mktemp -d -t tmp_find_duplicate_files.XXXXXX)" testdir2="$(cd tests/data/ && mktemp -d -t tmp_find_duplicate_files2.XXXXXX)" +# shellcheck disable=SC2064,SC2086 trap "rm -fr '$testdir1' '$testdir2'" $TRAP_SIGNALS echo test > "$testdir1/test1.txt" @@ -135,6 +137,8 @@ rm -fr "$testdir1" "$testdir2" echo echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Tests run: $run_count" time_taken "$start_time" "find_duplicate_files.py tests completed in" echo From 229de7ed0b9541ec0c918afbc02d98ae8dd7f1f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:00:03 +0000 Subject: [PATCH 0700/2779] updated test_getent.sh --- tests/test_getent.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_getent.sh b/tests/test_getent.sh index d2b87ddfe..26a7aa2b5 100755 --- a/tests/test_getent.sh +++ b/tests/test_getent.sh @@ -19,34 +19,38 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1091 . ./bash-tools/lib/utils.sh section "Getent" start_time="$(start_timer "find_active_server.py test")" -system=`uname -s` +system="$(uname -s)" echo "system = $system" hr -if [ "$system" = "Linux" -o "$system" = Darwin ]; then +if [ "$system" = "Linux" ] || + [ "$system" = Darwin ]; then run ./getent.py passwd | grep -v ':[x*!]*:' # counter is lost in subshell, increment manually run++ # $USER isn't always available in docker containers, use 'id' instead - run ./getent.py passwd `id -un` + run ./getent.py passwd "$(id -un)" run_fail 2 ./getent.py passwd nonexistentuser run ./getent.py group | grep -v ':[x*!]:' run++ - run ./getent.py group `id -gn` + run ./getent.py group "$(id -gn)" run_fail 2 ./getent.py group nonexistentgroup echo + # $run_count defined in lib + # shellcheck disable=SC2154 echo "Tests run: $run_count" time_taken "$start_time" "find_active_server.py tests completed in" else From ab829ff61b39faa1075a3aa7aaa595ab8333ad0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:01:23 +0000 Subject: [PATCH 0701/2779] updated test_hadoop.sh --- tests/test_hadoop.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_hadoop.sh b/tests/test_hadoop.sh index 25b705f8a..c2ebedfe9 100755 --- a/tests/test_hadoop.sh +++ b/tests/test_hadoop.sh @@ -19,12 +19,13 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$srcdir/.." +# shellcheck disable=SC1090 . "$srcdir/utils.sh" section "H a d o o p" # find_active_hadoop_namenode.py doesn't work on Hadoop 2.2 as the JMX bean isn't present -export HADOOP_VERSIONS="${@:-${HADOOP_VERSIONS:-latest 2.3 2.4 2.5 2.6 2.7 2.8}}" +export HADOOP_VERSIONS="${*:-${HADOOP_VERSIONS:-latest 2.3 2.4 2.5 2.6 2.7 2.8}}" HADOOP_HOST="${DOCKER_HOST:-${HADOOP_HOST:-${HOST:-localhost}}}" HADOOP_HOST="${HADOOP_HOST##*/}" @@ -61,6 +62,7 @@ test_hadoop(){ docker_compose_port HADOOP_YARN_NODE_MANAGER_PORT "Yarn NM" export HADOOP_PORTS="$HADOOP_NAMENODE_PORT $HADOOP_DATANODE_PORT $HADOOP_YARN_RESOURCE_MANAGER_PORT $HADOOP_YARN_NODE_MANAGER_PORT" hr + # shellcheck disable=SC2086 when_ports_available "$HADOOP_HOST" $HADOOP_PORTS hr # don't use the worker nodes so not testing for their availability @@ -125,10 +127,12 @@ EOFCOMMENTED # therefore reset the HADOOP PORTS to point to something that should get connection refused like port 1 and so that the failure hosts still fail and return only the expected correct host HADOOP_NAMENODE_PORT=1 ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_hadoop_namenode.py 127.0.0.2 127.0.0.3 "$HADOOP_HOST:$HADOOP_DATANODE_PORT" + # shellcheck disable=SC2097,SC2098 HADOOP_NAMENODE_PORT=1 run_grep "^$HADOOP_HOST:$HADOOP_NAMENODE_PORT$" ./find_active_hadoop_namenode.py 127.0.0.2 "$HADOOP_HOST:$HADOOP_DATANODE_PORT" 127.0.0.3 "$HADOOP_HOST:$HADOOP_NAMENODE_PORT" HADOOP_YARN_RESOURCE_MANAGER_PORT=1 ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_hadoop_yarn_resource_manager.py 127.0.0.2 127.0.0.3 "$HADOOP_HOST:$HADOOP_YARN_NODE_MANAGER_PORT" + # shellcheck disable=SC2097,SC2098 HADOOP_YARN_RESOURCE_MANAGER_PORT=1 run_grep "^$HADOOP_HOST:$HADOOP_YARN_RESOURCE_MANAGER_PORT$" ./find_active_hadoop_yarn_resource_manager.py 127.0.0.2 "$HADOOP_HOST:$HADOOP_YARN_NODE_MANAGER_PORT" 127.0.0.3 "$HADOOP_HOST:$HADOOP_YARN_RESOURCE_MANAGER_PORT" [ -z "${KEEPDOCKER:-}" ] || docker-compose down From 89d1389af075fa9ba5d1ebd3486cf48cef98dc9d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:06:26 +0000 Subject: [PATCH 0702/2779] updated test_hbase.sh --- tests/test_hbase.sh | 93 ++++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/tests/test_hbase.sh b/tests/test_hbase.sh index ebde33a96..28186c242 100755 --- a/tests/test_hbase.sh +++ b/tests/test_hbase.sh @@ -16,14 +16,15 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "$srcdir2/.." +cd "$srcdir/.." -. "$srcdir2/utils.sh" -. "$srcdir2/../bash-tools/lib/docker.sh" +# shellcheck disable=SC1090 +. "$srcdir/utils.sh" -srcdir="$srcdir2" +# shellcheck disable=SC1090 +. "$srcdir/../bash-tools/lib/docker.sh" section "H B a s e" @@ -39,7 +40,7 @@ export HBASE_THRIFT_PORT_DEFAULT=9090 export HBASE_THRIFT_UI_PORT_DEFAULT=9095 export ZOOKEEPER_PORT_DEFAULT=2181 -export HBASE_VERSIONS="${@:-latest 0.96 0.98 1.0 1.1 1.2 1.3}" +export HBASE_VERSIONS="${*:-latest 0.96 0.98 1.0 1.1 1.2 1.3}" check_docker_available @@ -59,9 +60,10 @@ test_hbase(){ fi VERSION="$version" docker-compose up -d hr - if [ "$version" = "0.96" -o "$version" = "0.98" ]; then - local export HBASE_MASTER_PORT_DEFAULT=60010 - local export HBASE_REGIONSERVER_PORT_DEFAULT=60301 + if [ "$version" = "0.96" ] || + [ "$version" = "0.98" ]; then + export HBASE_MASTER_PORT_DEFAULT=60010 + export HBASE_REGIONSERVER_PORT_DEFAULT=60301 fi echo "getting HBase dynamic port mappings:" docker_compose_port "HBase Master" @@ -73,6 +75,7 @@ test_hbase(){ #docker_compose_port ZOOKEEPER_PORT "HBase ZooKeeper" export HBASE_PORTS="$HBASE_MASTER_PORT $HBASE_REGIONSERVER_PORT $HBASE_STARGATE_PORT $HBASE_STARGATE_UI_PORT $HBASE_THRIFT_PORT $HBASE_THRIFT_UI_PORT" hr + # shellcheck disable=SC2086 when_ports_available "$HBASE_HOST" $HBASE_PORTS hr if [ "${version:0:3}" = "0.9" ]; then @@ -118,10 +121,12 @@ EOF return fi # will otherwise pick up HBASE_HOST and use default port and return the real HBase Master + # shellcheck disable=SC2097,SC2098 HBASE_HOST='' HOST='' HBASE_MASTER_PORT="$HBASE_MASTER_PORT_DEFAULT" \ ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_hbase_master.py 127.0.0.2 127.0.0.3 "$HBASE_HOST:$HBASE_REGIONSERVER_PORT" # if HBASE_PORT / --port is set to same as suffix then only outputs host not host:port + # shellcheck disable=SC2097,SC2098 HBASE_HOST='' HOST='' HBASE_MASTER_PORT="$HBASE_MASTER_PORT_DEFAULT" \ run_grep "^$HBASE_HOST:$HBASE_MASTER_PORT$" ./find_active_hbase_master.py 127.0.0.2 "$HBASE_HOST:$HBASE_REGIONSERVER_PORT" 127.0.0.3 "$HBASE_HOST:$HBASE_MASTER_PORT" @@ -245,53 +250,53 @@ EOF run_conn_refused ./hbase_table_row_key_distribution.py -T HexStringSplitTable # ============================================================================ # - run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 - run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 --average - run ./hbase_region_requests.py -T HexStringSplitTable $HBASE_HOST -c 2 --average --skip-zeros + run ./hbase_region_requests.py -T HexStringSplitTable "$HBASE_HOST" -c 2 + run ./hbase_region_requests.py -T HexStringSplitTable "$HBASE_HOST" -c 2 --average + run ./hbase_region_requests.py -T HexStringSplitTable "$HBASE_HOST" -c 2 --average --skip-zeros - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 --skip-zeros - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST -c 2 --average + run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" -c 2 + run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" -c 2 --skip-zeros + run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" -c 2 --average - run ./hbase_region_requests.py -T HS_test_data $HBASE_HOST --count 2 --interval 2 + run ./hbase_region_requests.py -T HS_test_data "$HBASE_HOST" --count 2 --interval 2 - run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 2 - run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST --count 2 -i 2 - run ./hbase_region_requests.py -T HS_test_data localhost $HBASE_HOST -c 2 --average + run ./hbase_region_requests.py -T HS_test_data localhost "$HBASE_HOST" -c 2 + run ./hbase_region_requests.py -T HS_test_data localhost "$HBASE_HOST" --count 2 -i 2 + run ./hbase_region_requests.py -T HS_test_data localhost "$HBASE_HOST" -c 2 --average # ============================================================================ # - run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 - run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 --average + run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 + run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 --average - run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 -T read,write,total - run ./hbase_regionserver_requests.py $HBASE_HOST -c 1 --type read,write,total --average + run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 -T read,write,total + run ./hbase_regionserver_requests.py "$HBASE_HOST" -c 1 --type read,write,total --average - run ./hbase_regionserver_requests.py $HBASE_HOST --count 2 --interval 2 + run ./hbase_regionserver_requests.py "$HBASE_HOST" --count 2 --interval 2 - run ./hbase_regionserver_requests.py localhost $HBASE_HOST -c 1 - run ./hbase_regionserver_requests.py localhost $HBASE_HOST --count 2 -i 2 - run ./hbase_regionserver_requests.py localhost $HBASE_HOST -c 1 --average + run ./hbase_regionserver_requests.py localhost "$HBASE_HOST" -c 1 + run ./hbase_regionserver_requests.py localhost "$HBASE_HOST" --count 2 -i 2 + run ./hbase_regionserver_requests.py localhost "$HBASE_HOST" -c 1 --average # ============================================================================ # - run ./hbase_regions_by_size.py $HBASE_HOST - run ./hbase_regions_by_size.py $HBASE_HOST --smallest - run ./hbase_regions_by_size.py $HBASE_HOST --human - run ./hbase_regions_by_size.py $HBASE_HOST --human -s - run ./hbase_regions_by_size.py $HBASE_HOST --human --top 10 - run ./hbase_regions_by_size.py $HBASE_HOST --human --top 10 --smallest - - run ./hbase_regions_by_memstore_size.py $HBASE_HOST - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --smallest - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human -s - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human --top 10 - run ./hbase_regions_by_memstore_size.py $HBASE_HOST --human --top 10 --smallest + run ./hbase_regions_by_size.py "$HBASE_HOST" + run ./hbase_regions_by_size.py "$HBASE_HOST" --smallest + run ./hbase_regions_by_size.py "$HBASE_HOST" --human + run ./hbase_regions_by_size.py "$HBASE_HOST" --human -s + run ./hbase_regions_by_size.py "$HBASE_HOST" --human --top 10 + run ./hbase_regions_by_size.py "$HBASE_HOST" --human --top 10 --smallest + + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --smallest + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human -s + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human --top 10 + run ./hbase_regions_by_memstore_size.py "$HBASE_HOST" --human --top 10 --smallest # ============================================================================ # - run ./hbase_regions_least_used.py $HBASE_HOST -r 20000 - run ./hbase_regions_least_used.py $HBASE_HOST -r 0 - run ./hbase_regions_least_used.py $HBASE_HOST --human --requests 20000 - run ./hbase_regions_least_used.py $HBASE_HOST --human --requests 20000 --top 10 + run ./hbase_regions_least_used.py "$HBASE_HOST" -r 20000 + run ./hbase_regions_least_used.py "$HBASE_HOST" -r 0 + run ./hbase_regions_least_used.py "$HBASE_HOST" --human --requests 20000 + run ./hbase_regions_least_used.py "$HBASE_HOST" --human --requests 20000 --top 10 [ -z "${KEEPDOCKER:-}" ] || docker-compose down From 1d0ff74fa9561121e2acf0a451cd77402d0a901f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:07:13 +0000 Subject: [PATCH 0703/2779] updated test_hexanonymize.sh --- tests/test_hexanonymize.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_hexanonymize.sh b/tests/test_hexanonymize.sh index 57da1e297..9477001b2 100755 --- a/tests/test_hexanonymize.sh +++ b/tests/test_hexanonymize.sh @@ -15,6 +15,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "HexAnonymize" @@ -34,7 +35,9 @@ run++ check_output "xyz123456rst789012abc" hexanonymize.py -o <<< "xyz987654rst654321caD" echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" -time_taken "$start_time" "All version tests for $name completed in" +time_taken "$start_time" "All version tests for hexanonymize.py completed in" echo untrap From 78f5b4bdd1e44edb836642ca258a9079e9d8a76c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:07:55 +0000 Subject: [PATCH 0704/2779] updated test_headtail.sh --- tests/test_headtail.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_headtail.sh b/tests/test_headtail.sh index 2ebb80933..8f7ceb05b 100755 --- a/tests/test_headtail.sh +++ b/tests/test_headtail.sh @@ -25,6 +25,7 @@ echo " cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh until [ $# -lt 1 ]; do @@ -34,7 +35,7 @@ until [ $# -lt 1 ]; do done data_dir="tests/data" -broken_dir="$data_dir/broken_json_data" +#broken_dir="$data_dir/broken_json_data" testfile="$data_dir/plant_catalog.xml" @@ -42,7 +43,7 @@ check(){ cmd="$1" expected="$2" msg="$3" - output="$(eval $cmd)" + output="$(eval "$cmd")" result="$(cksum <<< "$output")" echo -n "checking headtail $msg => " if [ "$result" = "$expected" ]; then @@ -52,7 +53,7 @@ check(){ echo echo "full output: " echo - eval $cmd + eval "$cmd" echo echo "cksum: $result" exit 1 From 0b95aa636e38e2d66a40417986f5052e91821144 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:08:38 +0000 Subject: [PATCH 0705/2779] updated test_json.sh --- tests/test_json.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_json.sh b/tests/test_json.sh index e7aafae63..9bc12780f 100755 --- a/tests/test_json.sh +++ b/tests/test_json.sh @@ -25,6 +25,7 @@ echo " cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh until [ $# -lt 1 ]; do From a452171b06cd5c1b803e39ed20d66596f9c5b145 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:09:19 +0000 Subject: [PATCH 0706/2779] updated test_json.sh --- tests/test_json.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_json.sh b/tests/test_json.sh index 9bc12780f..9e86b0984 100755 --- a/tests/test_json.sh +++ b/tests/test_json.sh @@ -36,9 +36,10 @@ done # ignore multi-line json data file for spark testing for jsonFile in $(find "${1:-.}" -iname '*.json' | - grep -v '/spark-.*-bin-hadoop.*/' | - grep -v 'multirecord.json' | - grep -v -e 'broken' -e 'error'); do + grep -v -e '/spark-.*-bin-hadoop.*/' \ + -e 'multirecord.json' \ + -e 'broken' \ + -e 'error'); do echo "testing json file: $jsonFile" python -mjson.tool < "$jsonFile" > /dev/null done From 8ddc03a543abc1aa9e7447d33140b7e60dc0b45f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:10:24 +0000 Subject: [PATCH 0707/2779] updated test_json_docs_to_bulk_multiline.sh --- tests/test_json_docs_to_bulk_multiline.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_json_docs_to_bulk_multiline.sh b/tests/test_json_docs_to_bulk_multiline.sh index 333688b75..2e81b392a 100755 --- a/tests/test_json_docs_to_bulk_multiline.sh +++ b/tests/test_json_docs_to_bulk_multiline.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "Testing json_docs_to_bulk_multiline.py" @@ -85,6 +86,7 @@ echo "testing stdin" ./json_docs_to_bulk_multiline.py - < "$data_dir/test.json" > "$stdout" ./json_docs_to_bulk_multiline.py < "$data_dir/test.json" > "$stdout" echo "testing stdin and file mix" +# shellcheck disable=SC2094 ./json_docs_to_bulk_multiline.py "$data_dir/test.json" - < "$data_dir/test.json" > "$stdout" # ================================================== @@ -102,10 +104,11 @@ check_broken(){ filename="$1" expected_exitcode="${2:-2}" set +e - ./json_docs_to_bulk_multiline.py "$filename" ${@:3} 2> "$stderr" > "$stdout" + # shellcheck disable=SC2086 + ./json_docs_to_bulk_multiline.py "$filename" ${*:3} 2> "$stderr" > "$stdout" exitcode=$? set -e - if [ $exitcode = $expected_exitcode ]; then + if [ $exitcode = "$expected_exitcode" ]; then echo "successfully detected broken json in '$filename', returned exit code $exitcode" echo #elif [ $exitcode != 0 ]; then From be26a228d4b37bb46e77e22aac951e0ba505b7d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:10:44 +0000 Subject: [PATCH 0708/2779] updated test_json_to_xml.sh --- tests/test_json_to_xml.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 320cb86e8..0b20f8609 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -19,7 +19,10 @@ srcdir="$(cd "$(dirname "$0")" && pwd)" cd "$srcdir"; +# shellcheck disable=SC1091 . utils.sh + +# shellcheck disable=SC1091 . ../bash-tools/lib/utils.sh section "JSON => XML" From c17419487c5665e682e96b83ea3d0887919a9ce2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:12:00 +0000 Subject: [PATCH 0709/2779] updated test_git_check_branches_upstream.sh --- tests/test_git_check_branches_upstream.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_git_check_branches_upstream.sh b/tests/test_git_check_branches_upstream.sh index 878529a01..35a818eb8 100755 --- a/tests/test_git_check_branches_upstream.sh +++ b/tests/test_git_check_branches_upstream.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Git check branches upstream" From e0a99def1b2bd6270ab5c970206a166ae311cf6a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:14:05 +0000 Subject: [PATCH 0710/2779] updated test_opentsdb.sh --- tests/test_opentsdb.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/test_opentsdb.sh b/tests/test_opentsdb.sh index 70ceb7e3d..ac98e609a 100755 --- a/tests/test_opentsdb.sh +++ b/tests/test_opentsdb.sh @@ -16,14 +16,12 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$srcdir/.." -cd "$srcdir2/.." - -. "$srcdir2/utils.sh" - -srcdir="$srcdir2" +# shellcheck disable=SC1090 +. "$srcdir/utils.sh" echo " # ============================================================================ # @@ -41,7 +39,7 @@ export ZOOKEEPER_PORT=2181 export OPENTSDB_PORTS="$ZOOKEEPER_PORT $HBASE_STARGATE_PORT 8085 $HBASE_THRIFT_PORT 9095 3000 16000 16010 16201 16301" export OPENTSDB_TEST_PORTS="$ZOOKEEPER_PORT $HBASE_THRIFT_PORT 3000" -export OPENTSDB_VERSIONS="${@:-latest}" +export OPENTSDB_VERSIONS="${*:-latest}" #export DOCKER_IMAGE="opower/opentsdb" #export DOCKER_IMAGE="petergrace/opentsdb-docker" @@ -68,11 +66,13 @@ generate_test_data(){ #chars="$(echo {A..Z} {a..z} {0..9})" chars=$(echo {A..Z} | tr -d ' ') ts="$(date '+%s')" + # shellcheck disable=SC2034 for x in {1..100}; do + # shellcheck disable=SC2034 for y in {1..1000}; do metric="metric${chars:$((RANDOM % ${#chars})):1}" for z in {1..5}; do - echo "ship${RANDOM:0:3} $(($ts + $RANDOM)) $RANDOM id=$metric crew=$z" + echo "ship${RANDOM:0:3} $((ts + RANDOM)) $RANDOM id=$metric crew=$z" done done done > "$DATA_FILE" @@ -111,6 +111,7 @@ made up error line EOF hr echo "testing from data file and STDIN at the same time:" + # shellcheck disable=SC2094 ./opentsdb_import_metric_distribution.py --key-prefix-length 7 "$DATA_FILE" - < "$DATA_FILE" hr @@ -119,7 +120,7 @@ EOF } for version in $OPENTSDB_VERSIONS; do - test_opentsdb $version + test_opentsdb "$version" done if [ -z "${NODELETE:-}" ]; then echo -n "removing test data: " From 8538d8bfb96b576c719d273ad69e0f354392b7d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:15:46 +0000 Subject: [PATCH 0711/2779] updated test_presto.sh --- tests/test_presto.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_presto.sh b/tests/test_presto.sh index 7b47934f5..ba9add032 100755 --- a/tests/test_presto.sh +++ b/tests/test_presto.sh @@ -19,12 +19,13 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$srcdir/.."; +# shellcheck disable=SC1091 . ./tests/utils.sh section "P r e s t o S Q L" export PRESTO_TERADATA_VERSIONS="latest 0.152 0.157 0.167 0.179" -export PRESTO_VERSIONS="${@:-${PRESTO_VERSIONS:-$PRESTO_TERADATA_VERSIONS}}" +export PRESTO_VERSIONS="${*:-${PRESTO_VERSIONS:-$PRESTO_TERADATA_VERSIONS}}" PRESTO_HOST="${DOCKER_HOST:-${PRESTO_HOST:-${HOST:-localhost}}}" PRESTO_HOST="${PRESTO_HOST##*/}" @@ -64,8 +65,8 @@ test_presto2(){ when_url_content "http://$PRESTO_HOST:$PRESTO_PORT/v1/service/presto/general" nodeId hr expected_version="$version" - if [ "$version" = "latest" -o \ - "$version" = "NODOCKER" ]; then + if [ "$version" = "latest" ] || + [ "$version" = "NODOCKER" ]; then if [ "$teradata_distribution" = 1 ]; then echo "latest version, fetching latest version from DockerHub master branch" expected_version="$(dockerhub_latest_version presto)" @@ -82,6 +83,7 @@ test_presto2(){ hr PRESTO_PORT="$PRESTO_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_presto_coordinator.py $non_presto_node1 $non_presto_node2 + # shellcheck disable=SC2097,SC2098 PRESTO_PORT="$PRESTO_PORT_DEFAULT" run_grep "^$PRESTO_HOST:$PRESTO_PORT$" ./find_active_presto_coordinator.py $non_presto_node1 "$PRESTO_HOST:$PRESTO_PORT" echo "Completed $run_count Presto tests" @@ -117,11 +119,12 @@ test_presto(){ fi done fi - if [ "$teradata_distribution" = "1" -a $facebook_only -eq 0 ]; then + if [ "$teradata_distribution" = "1" ] && + [ $facebook_only -eq 0 ]; then echo "Testing Teradata's Presto distribution version: '$version'" COMPOSE_FILE="$srcdir/docker/presto-docker-compose.yml" test_presto2 "$version" # must call this manually here as we're sneaking in an extra batch of tests that run_test_versions is generally not aware of - let total_run_count+=$run_count + ((total_run_count+=run_count)) # reset this so it can be used in test_presto to detect now testing Facebook teradata_distribution=0 fi From 80c0d309d8cfc3a238e4021bb7c1e32fb6df7477 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:16:43 +0000 Subject: [PATCH 0712/2779] updated test_quay_show_tags.sh --- tests/test_quay_show_tags.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_quay_show_tags.sh b/tests/test_quay_show_tags.sh index c9c55ad62..6765bc9e9 100755 --- a/tests/test_quay_show_tags.sh +++ b/tests/test_quay_show_tags.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Quay.io Show Tags" @@ -38,6 +40,8 @@ echo echo echo "All Quay Show Tags tests completed successfully" echo +# $run_count defined in lib +# shellcheck disable=SC2154 echo "Total Tests run: $run_count" time_taken "$start_time" "Quay Show Tags tests completed in" echo From 3f453cf509c37fd6d81feb982801511440f0cb6a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:17:44 +0000 Subject: [PATCH 0713/2779] updated test_solrcloud.sh --- tests/test_solrcloud.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/test_solrcloud.sh b/tests/test_solrcloud.sh index bafb97a47..31df34f43 100755 --- a/tests/test_solrcloud.sh +++ b/tests/test_solrcloud.sh @@ -15,17 +15,16 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir2="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "$srcdir2/.." +cd "$srcdir/.." +# shellcheck disable=SC1091 . ./tests/utils.sh -srcdir="$srcdir2" - section "S o l r C l o u d" -export SOLRCLOUD_VERSIONS="${@:-${SOLRCLOUD_VERSIONS:-latest 4.10 5.5 6.0 6.1 6.2 6.3 6.4 6.5 6.6}}" +export SOLRCLOUD_VERSIONS="${*:-${SOLRCLOUD_VERSIONS:-latest 4.10 5.5 6.0 6.1 6.2 6.3 6.4 6.5 6.6}}" SOLR_HOST="${DOCKER_HOST:-${SOLR_HOST:-${HOST:-localhost}}}" SOLR_HOST="${SOLR_HOST##*/}" @@ -46,11 +45,9 @@ trap_debug_env solr zookeeper test_solrcloud(){ local version="$1" # SolrCloud 4.x needs some different args / locations - if [ ${version:0:1} = 4 ]; then - four=true + if [ "${version:0:1}" = 4 ]; then export SOLR_COLLECTION="collection1" else - four="" export SOLR_COLLECTION="gettingstarted" fi section2 "Setting up SolrCloud $version docker test container" @@ -63,7 +60,8 @@ test_solrcloud(){ hr when_url_content "http://$SOLR_HOST:$SOLR_PORT/solr/" "Solr Admin" hr - local DOCKER_CONTAINER="$(docker-compose ps | sed -n '3s/ .*//p')" + local DOCKER_CONTAINER + DOCKER_CONTAINER="$(docker-compose ps | sed -n '3s/ .*//p')" echo "container is $DOCKER_CONTAINER" if [ -n "${NOTESTS:-}" ]; then exit 0 @@ -73,6 +71,7 @@ test_solrcloud(){ hr SOLR_PORT="$SOLR_PORT_DEFAULT" ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_solrcloud.py $non_solr_node1 $non_solr_node2 + # shellcheck disable=SC2097,SC2098 SOLR_PORT="$SOLR_PORT_DEFAULT" run_grep "^$SOLR_HOST:$SOLR_PORT$" ./find_active_solrcloud.py $non_solr_node1 $non_solr_node2 "$SOLR_HOST:$SOLR_PORT" docker-compose down From 8473a3acbbf825dfca772330683d971adc4a2de8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:18:14 +0000 Subject: [PATCH 0714/2779] updated test_serf_event_handler.sh --- tests/test_serf_event_handler.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_serf_event_handler.sh b/tests/test_serf_event_handler.sh index 00acd0fe7..eee228571 100755 --- a/tests/test_serf_event_handler.sh +++ b/tests/test_serf_event_handler.sh @@ -19,8 +19,10 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.." +# shellcheck disable=SC1091 . "tests/utils.sh" +# shellcheck disable=SC1091 . "bash-tools/lib/utils.sh" section "Testing Serf Event Handler" From 3650ae179993319b7266164cd3cd73684d5064b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:19:48 +0000 Subject: [PATCH 0715/2779] updated test_spark_csv_to_avro.sh --- tests/test_spark_csv_to_avro.sh | 41 ++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/tests/test_spark_csv_to_avro.sh b/tests/test_spark_csv_to_avro.sh index 05cf1bc17..dc43c31d8 100755 --- a/tests/test_spark_csv_to_avro.sh +++ b/tests/test_spark_csv_to_avro.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark CSV => Avro" @@ -29,9 +30,9 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}" +export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" # requires using spark-avro 3.0.0+ -#export SPARK_VERSIONS="${@:-2.0.0}" +#export SPARK_VERSIONS="${*:-2.0.0}" # don't support Spark <= 1.3 due to difference in databricks avro dependency for SPARK_VERSION in $SPARK_VERSIONS; do @@ -57,24 +58,36 @@ for SPARK_VERSION in $SPARK_VERSIONS; do # resolved, was due to Spark 1.4+ requiring pyspark-shell for PYSPARK_SUBMIT_ARGS rm -fr "test-header-$dir.avro" - ../spark_csv_to_avro.py -c data/header.csv --has-header -a "test-header-$dir.avro" && - echo "SUCCEEDED with header with Spark $SPARK_VERSION" || - { echo "FAILED with header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_avro.py -c data/header.csv --has-header -a "test-header-$dir.avro"; then + echo "SUCCEEDED with header with Spark $SPARK_VERSION" + else + echo "FAILED with header with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-header-schemaoverride-$dir.avro" - ../spark_csv_to_avro.py -c data/header.csv -a "test-header-schemaoverride-$dir.avro" --has-header -s Year:String,Make,Model,Length:float && - echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" || - { echo "FAILED with header and schema override with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_avro.py -c data/header.csv -a "test-header-schemaoverride-$dir.avro" --has-header -s Year:String,Make,Model,Length:float; then + echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" + else + echo "FAILED with header and schema override with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-noheader-$dir.avro" - ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length -a "test-noheader-$dir.avro" && - echo "SUCCEEDED with no header with Spark $SPARK_VERSION" || - { echo "FAILED with no header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length -a "test-noheader-$dir.avro"; then + echo "SUCCEEDED with no header with Spark $SPARK_VERSION" + else + echo "FAILED with no header with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-noheader-types-$dir.avro" - ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length:float -a "test-noheader-types-$dir.avro" && - echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" || - { echo "FAILED with no header and float type with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_avro.py -c data/test.csv -s Year:String,Make,Model,Length:float -a "test-noheader-types-$dir.avro"; then + echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" + else + echo "FAILED with no header and float type with Spark $SPARK_VERSION" + exit 1 + fi # if [ "$(cksum < "test-header-$dir.avro/part-r-00001.avro")" = "$(cksum < "test-noheader-$dir.avro/part-r-00001.avro")" ]; then # echo "SUCCESSFULLY compared noheader with explicit schema and mixed implicit/explicit string types to headered csv avro output" From c637d656112137c899cb57f66a1d1eca4b96e912 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:21:11 +0000 Subject: [PATCH 0716/2779] updated test_spark_csv_to_parquet.sh --- tests/test_spark_csv_to_parquet.sh | 39 ++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/tests/test_spark_csv_to_parquet.sh b/tests/test_spark_csv_to_parquet.sh index 1fc4b6b55..7f95341e1 100755 --- a/tests/test_spark_csv_to_parquet.sh +++ b/tests/test_spark_csv_to_parquet.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark CSV => Parquet" @@ -29,7 +30,7 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}" +export SPARK_VERSIONS="${*:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -54,24 +55,36 @@ for SPARK_VERSION in $SPARK_VERSIONS; do # resolved, was due to Spark 1.4+ requiring pyspark-shell for PYSPARK_SUBMIT_ARGS rm -fr "test-header-$dir.parquet" - ../spark_csv_to_parquet.py -c data/header.csv --has-header -p "test-header-$dir.parquet" && - echo "SUCCEEDED with header with Spark $SPARK_VERSION" || - { echo "FAILED with header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_parquet.py -c data/header.csv --has-header -p "test-header-$dir.parquet"; then + echo "SUCCEEDED with header with Spark $SPARK_VERSION" + else + echo "FAILED with header with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-header-schemaoverride-$dir.parquet" - ../spark_csv_to_parquet.py -c data/header.csv -p "test-header-schemaoverride-$dir.parquet" --has-header -s Year:String,Make,Model,Length:float && - echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" || - { echo "FAILED with header and schema override with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_parquet.py -c data/header.csv -p "test-header-schemaoverride-$dir.parquet" --has-header -s Year:String,Make,Model,Length:float; then + echo "SUCCEEDED with header and schema override with Spark $SPARK_VERSION" + else + echo "FAILED with header and schema override with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-noheader-$dir.parquet" - ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length -p "test-noheader-$dir.parquet" && - echo "SUCCEEDED with no header with Spark $SPARK_VERSION" || - { echo "FAILED with no header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length -p "test-noheader-$dir.parquet"; then + echo "SUCCEEDED with no header with Spark $SPARK_VERSION" + else + echo "FAILED with no header with Spark $SPARK_VERSION" + exit 1 + fi rm -fr "test-noheader-types-$dir.parquet" - ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length:float -p "test-noheader-types-$dir.parquet" && - echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" || - { echo "FAILED with no header and float type with Spark $SPARK_VERSION"; exit 1; } + if ../spark_csv_to_parquet.py -c data/test.csv -s Year:String,Make,Model,Length:float -p "test-noheader-types-$dir.parquet"; then + echo "SUCCEEDED with no header and float type with Spark $SPARK_VERSION" + else + echo "FAILED with no header and float type with Spark $SPARK_VERSION" + exit 1 + fi # if [ "$(cksum < "test-header-$dir.parquet/part-r-00001.parquet")" = "$(cksum < "test-noheader-$dir.parquet/part-r-00001.parquet")" ]; then # echo "SUCCESSFULLY compared noheader with explicit schema and mixed implicit/explicit string types to headered csv parquet output" From 536e02325ce6a7882aa9fb5d5f0922aabec582bd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:21:47 +0000 Subject: [PATCH 0717/2779] updated test_spark_json_to_avro.sh --- tests/test_spark_json_to_avro.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_spark_json_to_avro.sh b/tests/test_spark_json_to_avro.sh index 7c511eb3a..d6506a50c 100755 --- a/tests/test_spark_json_to_avro.sh +++ b/tests/test_spark_json_to_avro.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark JSON => Avro" @@ -30,9 +31,9 @@ if is_inside_docker; then fi # don't support Spark <= 1.3 due to difference in databricks avro dependency -export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}" +export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" # requires upgrade to spark-avro 3.0.0 -#export SPARK_VERSIONS="${@:-2.0.0}" +#export SPARK_VERSIONS="${*:-2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -57,9 +58,12 @@ for SPARK_VERSION in $SPARK_VERSIONS; do # resolved, was due to Spark 1.4+ requiring pyspark-shell for PYSPARK_SUBMIT_ARGS rm -fr "test-$dir.avro" - ../spark_json_to_avro.py -j data/multirecord.json -a "test-$dir.avro" && - echo "SUCCEEDED with header with Spark $SPARK_VERSION" || - { echo "FAILED with header with Spark $SPARK_VERSION"; exit 1; } + if ../spark_json_to_avro.py -j data/multirecord.json -a "test-$dir.avro"; then + echo "SUCCEEDED with header with Spark $SPARK_VERSION" + else + echo "FAILED with header with Spark $SPARK_VERSION" + exit 1 + fi #../spark_json_to_avro.py -j data/multirecord.json -a "test-$dir.avro" -s Year:String,Make,Model,Dimension.0.Length:float && # echo "SUCCEEDED with header with Spark $SPARK_VERSION" || From c1f982d207c4e46463483eaa60299af14e9f81d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:23:17 +0000 Subject: [PATCH 0718/2779] updated test_spark_json_to_parquet.sh --- tests/test_spark_json_to_parquet.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_spark_json_to_parquet.sh b/tests/test_spark_json_to_parquet.sh index 170631ba8..9a4a8bc37 100755 --- a/tests/test_spark_json_to_parquet.sh +++ b/tests/test_spark_json_to_parquet.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark JSON => Parquet" @@ -29,7 +30,7 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}" +export SPARK_VERSIONS="${*:-1.3.1 1.4.0 1.5.1 1.6.2 2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -50,8 +51,11 @@ for SPARK_VERSION in $SPARK_VERSIONS; do echo export SPARK_HOME="$dir" rm -fr "test-$dir.parquet" - ../spark_json_to_parquet.py -j data/multirecord.json -p "test-$dir.parquet" && - echo "SUCCEEDED with Spark $SPARK_VERSION" || - { echo "FAILED test with Spark $SPARK_VERSION"; exit 1; } + if ../spark_json_to_parquet.py -j data/multirecord.json -p "test-$dir.parquet"; then + echo "SUCCEEDED with Spark $SPARK_VERSION" + else + echo "FAILED test with Spark $SPARK_VERSION" + exit 1 + fi done echo "SUCCESS" From 2fe6cf02d1ab155348702fa6dd5fc479bd66a291 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:23:55 +0000 Subject: [PATCH 0719/2779] updated test_spark_z_avro_to_parquet.sh --- tests/test_spark_z_avro_to_parquet.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_spark_z_avro_to_parquet.sh b/tests/test_spark_z_avro_to_parquet.sh index ed20eb34a..02fc1ea26 100755 --- a/tests/test_spark_z_avro_to_parquet.sh +++ b/tests/test_spark_z_avro_to_parquet.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark Avro => Parquet" @@ -29,9 +30,9 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}" +export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" # requires using spark-avro 3.0.0+ -#export SPARK_VERSIONS="${@:-2.0.0}" +#export SPARK_VERSIONS="${*:-2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -52,8 +53,11 @@ for SPARK_VERSION in $SPARK_VERSIONS; do echo export SPARK_HOME="$dir" rm -fr "test-$dir.parquet" - ../spark_avro_to_parquet.py -a "test-header-$dir.avro" -p "test-$dir.parquet" && - echo "SUCCEEDED with Spark $SPARK_VERSION" || - { echo "FAILED test with Spark $SPARK_VERSION"; exit 1; } + if ../spark_avro_to_parquet.py -a "test-header-$dir.avro" -p "test-$dir.parquet"; then + echo "SUCCEEDED with Spark $SPARK_VERSION" + else + echo "FAILED test with Spark $SPARK_VERSION" + exit 1 + fi done echo "SUCCESS" From 973c91007dd934cf18c272a974c004d7e8ab1b24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 14:24:24 +0000 Subject: [PATCH 0720/2779] updated test_spark_z_parquet_to_avro.sh --- tests/test_spark_z_parquet_to_avro.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_spark_z_parquet_to_avro.sh b/tests/test_spark_z_parquet_to_avro.sh index be4f764f8..6e88f0cfc 100755 --- a/tests/test_spark_z_parquet_to_avro.sh +++ b/tests/test_spark_z_parquet_to_avro.sh @@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir" +# shellcheck disable=SC1091 . ./utils.sh section "Spark Parquet => Avro" @@ -29,9 +30,9 @@ if is_inside_docker; then exit 0 fi -export SPARK_VERSIONS="${@:-1.4.0 1.5.1 1.6.2}" +export SPARK_VERSIONS="${*:-1.4.0 1.5.1 1.6.2}" # requires using spark-avro 3.0.0+ -#export SPARK_VERSIONS="${@:-2.0.0}" +#export SPARK_VERSIONS="${*:-2.0.0}" for SPARK_VERSION in $SPARK_VERSIONS; do dir="spark-$SPARK_VERSION-bin-hadoop2.6" @@ -52,8 +53,11 @@ for SPARK_VERSION in $SPARK_VERSIONS; do echo export SPARK_HOME="$dir" rm -fr "test-$dir.avro" - ../spark_parquet_to_avro.py -p "test-$dir.parquet" -a "test-$dir.avro" && - echo "SUCCEEDED with Spark $SPARK_VERSION" || - { echo "FAILED test with Spark $SPARK_VERSION"; exit 1; } + if ../spark_parquet_to_avro.py -p "test-$dir.parquet" -a "test-$dir.avro"; then + echo "SUCCEEDED with Spark $SPARK_VERSION" + else + echo "FAILED test with Spark $SPARK_VERSION" + exit 1 + fi done echo "SUCCESS" From 3a6d704fd8cfe62b13f930972d34eb9a7b2a5114 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 22 Jan 2020 18:21:20 +0000 Subject: [PATCH 0721/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 829cf04b0..55da73ff4 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -514,6 +514,11 @@ dest[138]=' aws elb create-load-balancer --load-balancer-name Date: Wed, 22 Jan 2020 18:27:43 +0000 Subject: [PATCH 0722/2779] updated anonymize.py --- anonymize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/anonymize.py b/anonymize.py index 07bc730ac..e0904b498 100755 --- a/anonymize.py +++ b/anonymize.py @@ -306,6 +306,8 @@ def __init__(self): 'aws7': r'\bsg-[A-Za-z0-9]{8}(?. anyway by later fqdn anonymization + #'aws10': r'ec2-\d+-\d+-\d+-\d+\.{region}(\.compute\.amazonaws\.com)'.format(region='[A-Za-z0-9-]+'), 'db': r'({switch_prefix}(?:db|database)-?name{arg_sep})\S+'\ .format(arg_sep=arg_sep, switch_prefix=switch_prefix), @@ -449,6 +451,7 @@ def __init__(self): 'aws7': r'', 'aws8': r'\1:///', 'aws9': r'', + #'aws10': r'ec2-x-x-x-x.\1', 'db': r'\1', 'db2': r'\1', 'db3': r'\1', From a85da022d15d2b14653607e3c9e9a8743f221453 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 23 Jan 2020 13:13:06 +0000 Subject: [PATCH 0723/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 55da73ff4..cb781e11e 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -519,6 +519,9 @@ src[140]='ssh -i myKey -N -L 8888:ec2-1-2-3-4.eu-west-1.compute.amazonaws.com:88 #dest[140]='ssh -i myKey -N -L 8888::8888 @' dest[140]='ssh -i myKey -N -L 8888::8888 @' +src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-12345-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" +dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db/
/part-r-12345-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 7a857e497fafb1f43afa09b1532232d1a2754561 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 23 Jan 2020 13:13:08 +0000 Subject: [PATCH 0724/2779] added hive warehouse db/table match and tightened domain\user format to avoid newlines and carriage returns --- anonymize.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/anonymize.py b/anonymize.py index e0904b498..106f73952 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.8' +__version__ = '0.10.9' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -320,6 +320,7 @@ def __init__(self): id_or_name=id_or_name, switch_prefix=switch_prefix), 'db4': r'(\s(?:in|of)\s+(column|table|database|schema)\s+[\'"])[^\'"]+', + 'db5': r'/+user/+hive/+warehouse/+([A-Za-z0-9_-]+/+)*[A-Za-z0-9_-]+.db/+[A-Za-z0-9_-]+', 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex), 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\ .format(arg_sep=arg_sep, @@ -357,7 +358,7 @@ def __init__(self): 'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/(home|user)/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), - 'user4': r'(?/) exclude patterns '>/' where we have already matched and token replaced @@ -456,6 +457,7 @@ def __init__(self): 'db2': r'\1', 'db3': r'\1', 'db4': r'\1<\2>', + 'db5': r'/user/hive/warehouse/.db/
', 'generic': r'\1://', 'generic2': r'\1', 'generic3': r'\1', From 6f2269dd23ea84ca433a495b36283d70e07019c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 23 Jan 2020 13:17:36 +0000 Subject: [PATCH 0725/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index cb781e11e..919d91e95 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -519,8 +519,8 @@ src[140]='ssh -i myKey -N -L 8888:ec2-1-2-3-4.eu-west-1.compute.amazonaws.com:88 #dest[140]='ssh -i myKey -N -L 8888::8888 @' dest[140]='ssh -i myKey -N -L 8888::8888 @' -src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-12345-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" -dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db/
/part-r-12345-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" +src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" +dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db/
/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" # TODO: move proxy hosts to host matches and re-enable From 65237f5e3d264f61026ecb8666d6a8e026b508cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Jan 2020 12:21:52 +0000 Subject: [PATCH 0726/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 919d91e95..5f7657ea8 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -399,9 +399,9 @@ src[102]="-Dhost.domain.com=blah" dest[102]="-Dhost.domain.com=blah" # check escape codes get stripped if present (eg. if piping from grep --color-yes) -#src[88]="some^[[01;31m^[[Khost^[[m^[[Kname:443" -#src[88]="some\e[01;31m\e[Khost\e[m\e[K:443" -src[103]="$(echo somehost:443 | grep --color=yes host)" +# breaks test_anonymize.py which doesn't eval this, so put it explicitly +#src[103]="$(echo somehost:443 | grep --color=yes host)" +src[103]="somehost:443" dest[103]=":443" src[104]='..., "user": "blah", "group": "blah2", "host": "blah3", ...' From 082120243e8e4d65073748cd4b17f0b9e967b839 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Jan 2020 12:28:45 +0000 Subject: [PATCH 0727/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 116 ++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 5f7657ea8..72ded4847 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -40,40 +40,6 @@ start_time="$(start_timer "$anonymize")" # Custom Tests # ============================================================================ # -if [ -z "$test_nums" ]; then - echo - echo "Running Custom Tests:" - echo - echo "checking file args:" - run++ - if [ "$($anonymize -ae README.md | wc -l)" -gt 100 ]; then - echo "SUCCEEDED - anonymized README.md > 100 lines" - else - echo "FAILED - suspicious README.md file arg result came to <= 100 lines" - exit 1 - fi - hr - - run_grep "@" $anonymize --email <<< "hari@domain.com" - run_grep "@" $anonymize -E <<< "hari@domain.com" - - run_grep ".1" $anonymize -a --ip-prefix <<< "4.3.2.1" - run_grep ".1/" $anonymize --ip-prefix <<< "4.3.2.1/24" - run_grep ".1" $anonymize --ip-prefix <<< "4.3.2.1" - run_grep ".4" $anonymize --ip-prefix <<< "ip-1-2-3-4" - run_grep "ip-1-2-3-4-5" $anonymize --ip-prefix <<< "ip-1-2-3-4-5" - run_grep "dip-1-2-3-4" $anonymize --ip-prefix <<< "dip-1-2-3-4" - run_grep "5.4.3.2.1" $anonymize --ip-prefix <<< "5.4.3.2.1" - run_grep "log4j-1.2.3.4.jar" $anonymize --ip-prefix <<< "log4j-1.2.3.4.jar" - run_grep "/usr/hdp/2.6.2.0-123" $anonymize --ip-prefix <<< "/usr/hdp/2.6.2.0-123" - - run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path" - run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir' - run_grep '-host [a-f0-9]{12}' $anonymize --hash-hostnames <<< '-host blah' -fi - -# ============================================================================ # - src[0]="2015-11-19 09:59:59,893 - Execution of 'mysql -u root --password=somep@ssword! -h myHost.internal -s -e \"select version();\"' returned 1. ERROR 2003 (HY000): Can't connect to MySQL server on 'host.domain.com' (111)" dest[0]="2015-11-19 09:59:59,893 - Execution of 'mysql -u root --password= -h -s -e \"select version();\"' returned 1. ERROR 2003 (HY000): Can't connect to MySQL server on '' (111)" @@ -578,22 +544,20 @@ run_tests(){ for i in $test_numbers; do [ -n "${src[$i]:-}" ] || { echo "code error: src[$i] not defined"; exit 1; } [ -n "${dest[$i]:-}" ] || { echo "code error: dest[$i] not defined"; exit 1; } - if [ -n "$parallel" ]; then - test_anonymize "${src[$i]}" "${dest[$i]}" & - else - test_anonymize "${src[$i]}" "${dest[$i]}" - fi + #test_anonymize "${src[$i]}" "${dest[$i]}" + run++ done + "$srcdir/test_anonymize.py" } -echo -echo "Running Standard Tests with --all --skip-exceptions" -echo +#echo +#echo "Running Standard Tests with --all --skip-exceptions" +#echo run_tests # ignore_run_unqualified -echo -echo "Running Tests preseving text without --network enabled:" -echo +#echo +#echo "Running Tests preseving text without --network enabled:" +#echo # check normal don't strip these src[901]="reading password from foo" dest[901]="reading password from foo" @@ -602,11 +566,11 @@ src[902]="some description = blah, module = foo" dest[902]="some description = blah, module = foo" args="-HKEiu" -run_tests 901 902 # ignore_run_unqualified +#run_tests 901 902 # ignore_run_unqualified -echo -echo "Running Network Specific Tests:" -echo +#echo +#echo "Running Network Specific Tests:" +#echo # now check --network / --cisco / --juniper do strip these src[903]="reading password from bar" dest[903]="reading password " @@ -615,16 +579,50 @@ src[904]="some description = blah, module=bar" dest[904]="some description " args="--network" -run_tests 903 904 # ignore_run_unqualified - -if [ -n "$parallel" ]; then - # can't trust exit code for parallel yet, only for quick local testing - exit 1 -# for i in ${!src[@]}; do -# let j=$i+1 -# wait %$j -# [ $? -eq 0 ] || { echo "FAILED"; exit $?; } -# done +#run_tests 903 904 # ignore_run_unqualified + +#if [ -n "$parallel" ]; then +# # can't trust exit code for parallel yet, only for quick local testing +# exit 1 +## for i in ${!src[@]}; do +## let j=$i+1 +## wait %$j +## [ $? -eq 0 ] || { echo "FAILED"; exit $?; } +## done +#fi + +# ============================================================================ # + +if [ -z "$test_nums" ]; then + echo + echo "Running Custom Tests:" + echo + echo "checking file args:" + run++ + if [ "$($anonymize -ae README.md | wc -l)" -gt 100 ]; then + echo "SUCCEEDED - anonymized README.md > 100 lines" + else + echo "FAILED - suspicious README.md file arg result came to <= 100 lines" + exit 1 + fi + hr + + run_grep "@" $anonymize --email <<< "hari@domain.com" + run_grep "@" $anonymize -E <<< "hari@domain.com" + + run_grep ".1" $anonymize -a --ip-prefix <<< "4.3.2.1" + run_grep ".1/" $anonymize --ip-prefix <<< "4.3.2.1/24" + run_grep ".1" $anonymize --ip-prefix <<< "4.3.2.1" + run_grep ".4" $anonymize --ip-prefix <<< "ip-1-2-3-4" + run_grep "ip-1-2-3-4-5" $anonymize --ip-prefix <<< "ip-1-2-3-4-5" + run_grep "dip-1-2-3-4" $anonymize --ip-prefix <<< "dip-1-2-3-4" + run_grep "5.4.3.2.1" $anonymize --ip-prefix <<< "5.4.3.2.1" + run_grep "log4j-1.2.3.4.jar" $anonymize --ip-prefix <<< "log4j-1.2.3.4.jar" + run_grep "/usr/hdp/2.6.2.0-123" $anonymize --ip-prefix <<< "/usr/hdp/2.6.2.0-123" + + run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path" + run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir' + run_grep '-host [a-f0-9]{12}' $anonymize --hash-hostnames <<< '-host blah' fi echo From f9ec043ff08fba90769a492bf37298b4ea83b276 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Jan 2020 12:29:28 +0000 Subject: [PATCH 0728/2779] added tests/test_anonymize.py --- tests/test_anonymize.py | 80 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 tests/test_anonymize.py diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py new file mode 100755 index 000000000..8cddda652 --- /dev/null +++ b/tests/test_anonymize.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import re +#import StringIO +import subprocess +from subprocess import PIPE +import sys + +srcdir = os.path.abspath(os.path.dirname(__file__)) + +anonymize_test_sh = os.path.join(srcdir, 'test_anonymize.sh') + +anonymize = '{}/../anonymize.py'.format(srcdir) + +src = {} +dest = {} + +src_regex = re.compile(r'^\s*src\[(\d+)\]=["\'](.+)["\']\s*$') +dest_regex = re.compile(r'^\s*dest\[(\d+)\]=["\'](.+)["\']\s*$') +args_regex = re.compile(r'^args=["\'](.+)["\']\s*$') + +def normalize_text(text): + text = text.replace(r'\"', '"') + text = text.replace(r"\'", "'") + return text + +def run(): + #test_input = StringIO.StringIO() + #test_input.write('\n'.join(src)) + global src + global dest + src = {int(k) : v for k, v in src.items()} + dest = {int(k) : v for k, v in dest.items()} + src_keys = [key for key in sorted(src)] # pylint: disable=redefined-outer-name + test_input = '\n'.join([src[key] for key in src_keys]) + + print('running anonymize tests using: {} {}'.format(anonymize, args)) + process = subprocess.Popen([anonymize, args], stdin=PIPE, stdout=PIPE) + (stdout, _) = process.communicate(input=test_input) + index = 0 + for line in stdout.split('\n'): # pylint: disable=redefined-outer-name + key = src_keys[index] + _input = src[key] + expected = dest[key] + if line != expected: + print('FAILED to anonymize line during test {}'.format(key)) + print('input: {}'.format(_input)) + print('expected: {}'.format(expected)) + print('got: {}'.format(line)) + sys.exit(1) + print('SUCCEEDED anonymization test {}'.format(key)) + index += 1 + +with open(anonymize_test_sh) as filehandle: + for line in filehandle: + src_match = src_regex.match(line) + if src_match: + key = src_match.group(1) + if key in src: + raise AssertionError('Duplicate key index src[{}]'.format(key)) + value = src_match.group(2) + value = normalize_text(value) + src[key] = value + dest_match = dest_regex.match(line) + if dest_match: + key = dest_match.group(1) + if key in dest: + raise AssertionError('Duplicate key index dest[{}]'.format(key)) + value = dest_match.group(2) + value = normalize_text(value) + dest[key] = value + args_match = args_regex.match(line) + if args_match: + args = args_match.group(1) + run() + src = {} + dest = {} From b2259b32e0ebc9166aa55c8af9b1b8cf2581abb4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Jan 2020 12:30:02 +0000 Subject: [PATCH 0729/2779] updated test_anonymize.py --- tests/test_anonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index 8cddda652..c7a39e8c0 100755 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -30,8 +30,8 @@ def normalize_text(text): def run(): #test_input = StringIO.StringIO() #test_input.write('\n'.join(src)) - global src - global dest + global src # pylint: disable=global-statement + global dest # pylint: disable=global-statement src = {int(k) : v for k, v in src.items()} dest = {int(k) : v for k, v in dest.items()} src_keys = [key for key in sorted(src)] # pylint: disable=redefined-outer-name From 8a68421602fa8cd42fc8ef35e7abb9b78bac1eb5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 10:09:17 +0000 Subject: [PATCH 0730/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 72ded4847..d58f7315e 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -19,11 +19,11 @@ set -euo pipefail srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" test_nums="${*:-}" -parallel="" -if [ "$test_nums" = "p" ]; then - parallel="1" - test_nums="" -fi +#parallel="" +#if [ "$test_nums" = "p" ]; then +# parallel="1" +# test_nums="" +#fi cd "$srcdir/.."; @@ -553,7 +553,7 @@ run_tests(){ #echo #echo "Running Standard Tests with --all --skip-exceptions" #echo -run_tests # ignore_run_unqualified +run_tests "$@" # ignore_run_unqualified #echo #echo "Running Tests preseving text without --network enabled:" From 0fded27273ba1071f663e17b8d9625c6db6feb33 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 10:21:41 +0000 Subject: [PATCH 0731/2779] updated test_anonymize.py --- tests/test_anonymize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index c7a39e8c0..8d67fcf36 100755 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -20,7 +20,7 @@ src_regex = re.compile(r'^\s*src\[(\d+)\]=["\'](.+)["\']\s*$') dest_regex = re.compile(r'^\s*dest\[(\d+)\]=["\'](.+)["\']\s*$') -args_regex = re.compile(r'^args=["\'](.+)["\']\s*$') +args_regex = re.compile(r'^\s*args=["\'](.+)["\']\s*$') def normalize_text(text): text = text.replace(r'\"', '"') @@ -38,7 +38,8 @@ def run(): test_input = '\n'.join([src[key] for key in src_keys]) print('running anonymize tests using: {} {}'.format(anonymize, args)) - process = subprocess.Popen([anonymize, args], stdin=PIPE, stdout=PIPE) + cmd = [anonymize] + [_ for _ in args.split()] + process = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE) (stdout, _) = process.communicate(input=test_input) index = 0 for line in stdout.split('\n'): # pylint: disable=redefined-outer-name From 5991a1432fb3b56f3bc0b64a3626714e6787f95a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 10:24:52 +0000 Subject: [PATCH 0732/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index d58f7315e..7335c0168 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -610,15 +610,33 @@ if [ -z "$test_nums" ]; then run_grep "@" $anonymize --email <<< "hari@domain.com" run_grep "@" $anonymize -E <<< "hari@domain.com" - run_grep ".1" $anonymize -a --ip-prefix <<< "4.3.2.1" - run_grep ".1/" $anonymize --ip-prefix <<< "4.3.2.1/24" - run_grep ".1" $anonymize --ip-prefix <<< "4.3.2.1" - run_grep ".4" $anonymize --ip-prefix <<< "ip-1-2-3-4" - run_grep "ip-1-2-3-4-5" $anonymize --ip-prefix <<< "ip-1-2-3-4-5" - run_grep "dip-1-2-3-4" $anonymize --ip-prefix <<< "dip-1-2-3-4" - run_grep "5.4.3.2.1" $anonymize --ip-prefix <<< "5.4.3.2.1" - run_grep "log4j-1.2.3.4.jar" $anonymize --ip-prefix <<< "log4j-1.2.3.4.jar" - run_grep "/usr/hdp/2.6.2.0-123" $anonymize --ip-prefix <<< "/usr/hdp/2.6.2.0-123" + src[800]="4.3.2.1" + dest[800]=".1" + + src[801]="4.3.2.1/24" + dest[801]=".1/" + + src[802]="4.3.2.1" + dest[802]=".1" + + src[803]="ip-1-2-3-4" + dest[803]=".4" + + src[804]="ip-1-2-3-4-5" + dest[804]="ip-1-2-3-4-5" + + src[805]="dip-1-2-3-4" + dest[805]="dip-1-2-3-4" + + src[806]="5.4.3.2.1" + dest[806]="5.4.3.2.1" + + src[807]="log4j-1.2.3.4.jar" + dest[807]="log4j-1.2.3.4.jar" + + src[808]="/usr/hdp/2.6.2.0-123" + dest[808]="/usr/hdp/2.6.2.0-123" + args="-a --ip-prefix" run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path" run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir' From 364cf4e9fe09730ccd7d13fef16c7d394340cc2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:13:00 +0000 Subject: [PATCH 0733/2779] integrated with CLI for better option handling and logging --- hive_schemas_csv.py | 277 ++++++++++++++++++++++---------------------- 1 file changed, 137 insertions(+), 140 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index f93bb9137..e08b5414d 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -56,154 +56,151 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from __future__ import unicode_literals +#from __future__ import unicode_literals -import argparse import csv -import logging import os import socket import sys from impala.dbapi import connect +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_host, validate_port + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - name = 'HiveServer2' - default_port = 10000 - default_service_name = 'hive' - host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' - ] - port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' - ] - - if 'impala' in sys.argv[0]: - name = 'Impala' - default_port = 21050 - default_service_name = 'impala' - host_envs = [ - 'IMPALA_HOST', - 'HOST' - ] - port_envs = [ - 'IMPALA_PORT', - 'PORT' - ] - parser = argparse.ArgumentParser( - description="Dumps all {} schemas, tables, columns and types to CSV format on stdout".format(name)) - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='{} host '.format(name) + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), - help='{} port (default: {}, '.format(name, default_port) + \ - ', $'.join(port_envs) + ')') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default=default_service_name, - help='Service principal (default: {})'.format(default_service_name)) - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') - # must set type to str otherwise csv module gives this error on Python 2.7: - # TypeError: "delimiter" must be string, not unicode - parser.add_argument('-d', '--delimiter', default=',', type=str, help='Delimiter to use (default: ,)') - parser.add_argument('-Q', '--quotechar', default='"', type=str, - help='Generate quoted CSV (recommended, default is double quote \'"\')') - parser.add_argument('-E', '--escapechar', help='Escape char if needed') - parser.add_argument('-v', '--verbose', action='count', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - log.debug('kerberos enabled') - log.debug('krb5 remote service principal name = %s', args.krb5_service_name) - if args.ssl is True: - log.debug('ssl enabled') - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - conn = connect_db(args, 'default') - - quoting = csv.QUOTE_ALL - if args.quotechar == '': - quoting = csv.QUOTE_NONE - fieldnames = ['database', 'table', 'column', 'type'] - csv_writer = csv.DictWriter(sys.stdout, - delimiter=args.delimiter, - quotechar=args.quotechar, - escapechar=args.escapechar, - quoting=quoting, - fieldnames=fieldnames) - csv_writer.writeheader() - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - for table_row in table_cursor: - table = table_row[0] - log.info('describing table %s', table) - with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec - #column_cursor.execute('use %(database)s', {'database': database}) - #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use {}'.format(database)) - column_cursor.execute('describe {}'.format(table)) - for column_row in column_cursor: - column = column_row[0] - column_type = column_row[1] - csv_writer.writerow({'database': database, - 'table': table, - 'column': column, - 'type': column_type}) +__version__ = '0.4.0' + + +class HiveSchemasCSV(CLI): + + def __init__(self): + # Python 2.x + super(HiveSchemasCSV, self).__init__() + # Python 3.x + # super().__init__() + self.name = ['HiveServer2', 'Hive'] + self.host = None + self.port = None + self.default_host = socket.getfqdn() + self.default_port = 10000 + self.default_service_name = 'hive' + self.kerberos = False + self.krb5_service_name = self.default_service_name + self.ssl = False + self.delimiter = None + self.quotechar = None + self.escapechar = None + + if 'impala' in sys.argv[0]: + self.name = 'Impala' + self.default_port = 21050 + self.default_service_name = 'impala' + self.env_prefixes = ['IMPALA'] + + def add_options(self): + super(HiveSchemasCSV, self).add_options() + self.add_hostoption() + self.add_opt('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + self.add_opt('-n', '--krb5-service-name', default=self.default_service_name, + help='Service principal (default: {})'.format(self.default_service_name)) + self.add_opt('-S', '--ssl', action='store_true', help='Use SSL') + # must set type to str otherwise csv module gives this error on Python 2.7: + # TypeError: "delimiter" must be string, not unicode + # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals' + # breaks this - might break in Python 3 if the impyla module doesn't fix behaviour + self.add_opt('-d', '--delimiter', default=',', type=str, help='Delimiter to use (default: ,)') + self.add_opt('-Q', '--quotechar', default='"', type=str, + help='Generate quoted CSV (recommended, default is double quote \'"\')') + self.add_opt('-E', '--escapechar', help='Escape char if needed') + + def process_options(self): + super(HiveSchemasCSV, self).process_options() + self.host = self.get_opt('host') + self.port = self.get_opt('port') + validate_host(self.host) + validate_port(self.port) + self.port = int(self.port) + self.kerberos = self.get_opt('kerberos') + self.krb5_service_name = self.get_opt('krb5_service_name') + self.ssl = self.get_opt('ssl') + self.delimiter = self.get_opt('delimiter') + self.quotechar = self.get_opt('quotechar') + self.escapechar = self.get_opt('escapechar') + + def connect(self, database): + auth_mechanism = None + if self.kerberos: + auth_mechanism = 'GSSAPI' + log.debug('kerberos enabled') + log.debug('krb5 remote service principal name = %s', self.krb5_service_name) + if self.ssl: + log.debug('ssl enabled') + + log.info('connecting to %s:%s database %s', self.host, self.port, database) + return connect( + host=self.host, + port=self.port, + auth_mechanism=auth_mechanism, + use_ssl=self.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=self.krb5_service_name + ) + + def run(self): + + conn = self.connect('default') + + quoting = csv.QUOTE_ALL + if self.quotechar == '': + quoting = csv.QUOTE_NONE + fieldnames = ['database', 'table', 'column', 'type'] + csv_writer = csv.DictWriter(sys.stdout, + delimiter=self.delimiter, + quotechar=self.quotechar, + escapechar=self.escapechar, + quoting=quoting, + fieldnames=fieldnames) + csv_writer.writeheader() + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + log.info('describing table %s', table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use {}'.format(database)) + column_cursor.execute('describe {}'.format(table)) + for column_row in column_cursor: + column = column_row[0] + column_type = column_row[1] + csv_writer.writerow({'database': database, + 'table': table, + 'column': column, + 'type': column_type}) if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) + HiveSchemasCSV().main() From 031ea7c943c493ecb80f5a2b825d036799a1978e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:18:12 +0000 Subject: [PATCH 0734/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index e08b5414d..44d64c981 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -99,12 +99,6 @@ def __init__(self): self.quotechar = None self.escapechar = None - if 'impala' in sys.argv[0]: - self.name = 'Impala' - self.default_port = 21050 - self.default_service_name = 'impala' - self.env_prefixes = ['IMPALA'] - def add_options(self): super(HiveSchemasCSV, self).add_options() self.add_hostoption() From 737b4a1849ab169450850015e745dd538c028339 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:18:33 +0000 Subject: [PATCH 0735/2779] subclassed impala_schemas_csv.py --- impala_schemas_csv.py | 81 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) mode change 120000 => 100755 impala_schemas_csv.py diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py deleted file mode 120000 index baed3c963..000000000 --- a/impala_schemas_csv.py +++ /dev/null @@ -1 +0,0 @@ -hive_schemas_csv.py \ No newline at end of file diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py new file mode 100755 index 000000000..e6ac0ef90 --- /dev/null +++ b/impala_schemas_csv.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout + +In practice Hive is much more reliable for dumping masses of schema + +Impala appears faster initially but then slows down more than Hive and hits things query handle errors +under sustained load of extracting large amounts of schema information + +There is also a risk that Impala's metadata may be out of date, so Hive is strongly preferred for this + + +CSV format: + +database,table,column,type + + +I recommend generating quoted csv because you may encounter Hive data types such as decimal(15,2) +which would cause incorrect field splitting, you can disable by setting --quotechar='' to blank but +if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will +raise a traceback to tell you to set one (eg. --escapechar='\\') + +Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +from hive_schemas_csv import HiveSchemasCSV + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaSchemasCSV(HiveSchemasCSV): + + def __init__(self): + # Python 2.x + super(ImpalaSchemasCSV, self).__init__() + # Python 3.x + # super().__init__() + self.name = 'Impala' + self.default_port = 21050 + self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaSchemasCSV().main() From 70c9929895084a2cf835bb66b386e8b8d11f35d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:45:14 +0000 Subject: [PATCH 0736/2779] updated impala_schemas_csv.py --- impala_schemas_csv.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py index e6ac0ef90..2b6b19fdc 100755 --- a/impala_schemas_csv.py +++ b/impala_schemas_csv.py @@ -18,7 +18,7 @@ Connect to an Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout -In practice Hive is much more reliable for dumping masses of schema +In practice Hive is much more reliable than Impala for dumping masses of schema Impala appears faster initially but then slows down more than Hive and hits things query handle errors under sustained load of extracting large amounts of schema information @@ -61,7 +61,7 @@ from hive_schemas_csv import HiveSchemasCSV __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.5.0' class ImpalaSchemasCSV(HiveSchemasCSV): @@ -71,9 +71,11 @@ def __init__(self): super(ImpalaSchemasCSV, self).__init__() # Python 3.x # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class self.name = 'Impala' - self.default_port = 21050 - self.default_service_name = 'impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' if __name__ == '__main__': From 93d95469797dc051d4ae8813f3c5577119828698 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:45:39 +0000 Subject: [PATCH 0737/2779] refactored out to HiveImpalaCLI --- hive_schemas_csv.py | 64 ++++++++------------------------------------- 1 file changed, 11 insertions(+), 53 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 44d64c981..5b8d6c3eb 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -16,9 +16,9 @@ """ -Connect to a HiveServer2 or Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout +Connect to HiveServer2 and dump all the schemas, tables and columns out in CSV format to stdout -In practice Hive is much more reliable for dumping masses of schema +In practice Hive is much more reliable than Impala for dumping masses of schema Impala appears faster initially but then slows down more than Hive and hits things query handle errors under sustained load of extracting large amounts of schema information @@ -60,15 +60,16 @@ import csv import os -import socket import sys -from impala.dbapi import connect -libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) -sys.path.append(libdir) +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log, validate_host, validate_port - from harisekhon import CLI + from harisekhon.utils import log + from hive_impala_cli import HiveImpalaCLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) print("Did you remember to build the project by running 'make'?", file=sys.stderr) @@ -76,36 +77,22 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.5.0' -class HiveSchemasCSV(CLI): +class HiveSchemasCSV(HiveImpalaCLI): def __init__(self): # Python 2.x super(HiveSchemasCSV, self).__init__() # Python 3.x # super().__init__() - self.name = ['HiveServer2', 'Hive'] - self.host = None - self.port = None - self.default_host = socket.getfqdn() - self.default_port = 10000 - self.default_service_name = 'hive' - self.kerberos = False - self.krb5_service_name = self.default_service_name - self.ssl = False self.delimiter = None self.quotechar = None self.escapechar = None def add_options(self): super(HiveSchemasCSV, self).add_options() - self.add_hostoption() - self.add_opt('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - self.add_opt('-n', '--krb5-service-name', default=self.default_service_name, - help='Service principal (default: {})'.format(self.default_service_name)) - self.add_opt('-S', '--ssl', action='store_true', help='Use SSL') # must set type to str otherwise csv module gives this error on Python 2.7: # TypeError: "delimiter" must be string, not unicode # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals' @@ -117,39 +104,10 @@ def add_options(self): def process_options(self): super(HiveSchemasCSV, self).process_options() - self.host = self.get_opt('host') - self.port = self.get_opt('port') - validate_host(self.host) - validate_port(self.port) - self.port = int(self.port) - self.kerberos = self.get_opt('kerberos') - self.krb5_service_name = self.get_opt('krb5_service_name') - self.ssl = self.get_opt('ssl') self.delimiter = self.get_opt('delimiter') self.quotechar = self.get_opt('quotechar') self.escapechar = self.get_opt('escapechar') - def connect(self, database): - auth_mechanism = None - if self.kerberos: - auth_mechanism = 'GSSAPI' - log.debug('kerberos enabled') - log.debug('krb5 remote service principal name = %s', self.krb5_service_name) - if self.ssl: - log.debug('ssl enabled') - - log.info('connecting to %s:%s database %s', self.host, self.port, database) - return connect( - host=self.host, - port=self.port, - auth_mechanism=auth_mechanism, - use_ssl=self.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=self.krb5_service_name - ) - def run(self): conn = self.connect('default') From 75905a0b7937262dce1820190ddf8bb7495c01c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:49:54 +0000 Subject: [PATCH 0738/2779] updated .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f0ca6569a..3722dc711 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +#lib/ # breaks local lib/ change tracking lib64/ parts/ sdist/ From b2de84ef84db86f4cd5d54a273d7c7062782a60a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:50:01 +0000 Subject: [PATCH 0739/2779] added lib/hive_impala_cli.py --- lib/hive_impala_cli.py | 103 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 lib/hive_impala_cli.py diff --git a/lib/hive_impala_cli.py b/lib/hive_impala_cli.py new file mode 100755 index 000000000..d5f9a2a06 --- /dev/null +++ b/lib/hive_impala_cli.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import socket +import sys +from impala.dbapi import connect +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_host, validate_port + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveImpalaCLI(CLI): + + def __init__(self): + # Python 2.x + super(HiveImpalaCLI, self).__init__() + # Python 3.x + # super().__init__() + self.name = ['HiveServer2', 'Hive'] + self.host = None + self.port = None + self.default_host = socket.getfqdn() + self.default_port = 10000 + self.default_service_name = 'hive' + self.kerberos = False + self.krb5_service_name = self.default_service_name + self.ssl = False + #self.timeout_default = 86400 + self.timeout_default = None + if 'impala' in sys.argv[0]: + self.name = 'Impala' + self.default_port = 21050 + self.default_service_name = 'impala' + + def add_options(self): + super(HiveImpalaCLI, self).add_options() + self.add_hostoption() + self.add_opt('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + self.add_opt('-n', '--krb5-service-name', default=self.default_service_name, + help='Service principal (default: {})'.format(self.default_service_name)) + self.add_opt('-S', '--ssl', action='store_true', help='Use SSL') + + def process_options(self): + super(HiveImpalaCLI, self).process_options() + self.host = self.get_opt('host') + self.port = self.get_opt('port') + validate_host(self.host) + validate_port(self.port) + self.port = int(self.port) + self.kerberos = self.get_opt('kerberos') + self.krb5_service_name = self.get_opt('krb5_service_name') + self.ssl = self.get_opt('ssl') + + def connect(self, database): + auth_mechanism = None + if self.kerberos: + auth_mechanism = 'GSSAPI' + log.debug('kerberos enabled') + log.debug('krb5 remote service principal name = %s', self.krb5_service_name) + if self.ssl: + log.debug('ssl enabled') + + log.info('connecting to %s:%s database %s', self.host, self.port, database) + return connect( + host=self.host, + port=self.port, + auth_mechanism=auth_mechanism, + use_ssl=self.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=self.krb5_service_name + ) From 64cf017b4329bd99e282bb548426d99c5ee23ad8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:52:33 +0000 Subject: [PATCH 0740/2779] updated impala_schemas_csv.py --- impala_schemas_csv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py index 2b6b19fdc..17ce8987b 100755 --- a/impala_schemas_csv.py +++ b/impala_schemas_csv.py @@ -18,7 +18,7 @@ Connect to an Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout -In practice Hive is much more reliable than Impala for dumping masses of schema +In practice Hive is much more reliable than Impala for dumping masses of schema (see adjacent hive_schemas_csv.py) Impala appears faster initially but then slows down more than Hive and hits things query handle errors under sustained load of extracting large amounts of schema information @@ -36,7 +36,7 @@ if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will raise a traceback to tell you to set one (eg. --escapechar='\\') -Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From f2878bdbf4f1fd47c6ca4b9e2634a8d3f51cd3e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 15:52:49 +0000 Subject: [PATCH 0741/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 5b8d6c3eb..efdf62143 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -36,7 +36,7 @@ if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will raise a traceback to tell you to set one (eg. --escapechar='\\') -Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 216e44ab452e86b76f5845ad408c4f281f85eac5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:12:57 +0000 Subject: [PATCH 0742/2779] refactored to use HiveImpalaCLI --- hive_tables_row_counts.py | 314 ++++++++++++++++---------------------- 1 file changed, 132 insertions(+), 182 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index d4d685f26..08eefbfff 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -16,10 +16,10 @@ """ -Connect to a HiveServer2 / Impala node and get rows counts for all tables in all databases, +Connect to HiveServer2 and get rows counts for all tables in all databases, or only those matching given db / table / partition value regexes -Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos +Tested on Hive 1.1.0 CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see @@ -41,71 +41,46 @@ from __future__ import print_function from __future__ import unicode_literals -import argparse -import logging import os import re -import socket import sys import impala -from impala.dbapi import connect +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_regex + from hive_impala_cli import HiveImpalaCLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - name = 'HiveServer2' - default_port = 10000 - default_service_name = 'hive' - host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' - ] - port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' - ] - - if 'impala' in sys.argv[0]: - name = 'Impala' - default_port = 21050 - default_service_name = 'impala' - host_envs = [ - 'IMPALA_HOST', - 'HOST' - ] - port_envs = [ - 'IMPALA_PORT', - 'PORT' - ] - parser = argparse.ArgumentParser( - description="Gets row counts for all {} tables / partitions matching database / table / partition regexes"\ - .format(name)) - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='{} host '.format(name) + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), - help='{} port (default: {}, '.format(name, default_port) + \ - ', $'.join(port_envs) + ')') - parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') - parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') - parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default=default_service_name, - help='Service principal (default: {})'.format(default_service_name)) - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') +__version__ = '0.5.0' + + +class HiveTablesRowCounts(HiveImpalaCLI): + + def __init__(self): + # Python 2.x + super(HiveTablesRowCounts, self).__init__() + # Python 3.x + # super().__init__() + self.database = None + self.table = None + self.partition = None + self.ignore_errors = False + + def add_options(self): + super(HiveTablesRowCounts, self).add_options() + self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)') + self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)') + self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)') # # ignore tables that fail with errors like: # @@ -118,130 +93,105 @@ def parse_args(): # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' # - parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - try: - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - partition_regex = re.compile(args.partition, re.I) - except re.error as _: - log.error('error in provided regex: %s', _) - sys.exit(3) - - conn = connect_db(args, 'default') - - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - if not database_regex.search(database): - log.debug("skipping database '%s', does not match regex '%s'", database, args.database) - continue - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, args.table) - continue + self.add_opt('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue') + + def process_options(self): + super(HiveTablesRowCounts, self).process_options() + self.database = self.get_opt('database') + self.table = self.get_opt('table') + self.partition = self.get_opt('partition') + self.ignore_errors = self.get_opt('ignore_errors') + validate_regex(self.database, 'database') + validate_regex(self.table, 'table') + validate_regex(self.partition, 'partition') + + def run(self): + database_regex = re.compile(self.database, re.I) + table_regex = re.compile(self.table, re.I) + partition_regex = re.compile(self.partition, re.I) + conn = self.connect('default') + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, self.database) + continue + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: try: - get_row_counts(conn, args, database, table, partition_regex) - except Exception as _: - # invalid query handle and similar errors happen at higher level - # as they are not query specific, will not be caught here so still error out - if args.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): continue raise - -def get_row_counts(conn, args, database, table, partition_regex): - log.info("getting partitions for database '%s' table '%s'", database, table) - with conn.cursor() as partition_cursor: - # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('use {db}'.format(db=database)) - try: - partition_cursor.execute('show partitions {table}'.format(table=table)) - for partitions_row in partition_cursor: - partition_key = partitions_row[0] - partition_value = partitions_row[1] - if not partition_regex.match(partition_value): - log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + - "value does not match regex '%s'", - database, - table, - partition_key, - partition_value, - args.partition) - continue - # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ - .format(db=database, table=table, key=partition_key, value=partition_value)) - for result in partition_cursor: - row_count = result[0] - print('{db}.{table}.{key}={value}\t{row_count}'.format(\ - db=database, table=table, key=partition_key, value=partition_value, row_count=row_count)) - except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - # Hive impala.error.HiveServer2Error: is not a partitioned table - # Impala impala.error.HiveServer2Error: Table is not partitioned - if 'is not a partitioned table' not in str(_) and \ - 'Table is not partitioned' not in str(_): - raise - log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", - database, table) - with conn.cursor() as table_cursor: - log.info("running SELECT COUNT(*) FROM %s.%s", database, table) - # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) - for result in table_cursor: - row_count = result[0] - print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + try: + self.get_row_counts(conn, database, table, partition_regex) + except Exception as _: + # invalid query handle and similar errors happen at higher level + # as they are not query specific, will not be caught here so still error out + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise + + def get_row_counts(self, conn, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {db}'.format(db=database)) + try: + partition_cursor.execute('show partitions {table}'.format(table=table)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + if not partition_regex.match(partition_value): + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + self.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + for result in partition_cursor: + row_count = result[0] + print('{db}.{table}.{key}={value}\t{row_count}'.format(\ + db=database, + table=table, + key=partition_key, + value=partition_value, + row_count=row_count)) + except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + # Hive impala.error.HiveServer2Error: is not a partitioned table + # Impala impala.error.HiveServer2Error: Table is not partitioned + if 'is not a partitioned table' not in str(_) and \ + 'Table is not partitioned' not in str(_): + raise + log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", + database, table) + with conn.cursor() as table_cursor: + log.info("running SELECT COUNT(*) FROM %s.%s", database, table) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) + for result in table_cursor: + row_count = result[0] + print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) + HiveTablesRowCounts().main() From 1bb4945dfdb66d0279f21e4986de85ff21019c16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:13:25 +0000 Subject: [PATCH 0743/2779] changed to subclass HiveTablesRowCounts --- impala_tables_row_counts.py | 1 - 1 file changed, 1 deletion(-) delete mode 120000 impala_tables_row_counts.py diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py deleted file mode 120000 index 65e53c4f2..000000000 --- a/impala_tables_row_counts.py +++ /dev/null @@ -1 +0,0 @@ -hive_tables_row_counts.py \ No newline at end of file From d033db228338a6ca6c0763b29edac9a98bb18633 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:16:02 +0000 Subject: [PATCH 0744/2779] subclassed HiveTablesRowCounts --- impala_tables_row_counts.py | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 impala_tables_row_counts.py diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py new file mode 100755 index 000000000..e17a88e63 --- /dev/null +++ b/impala_tables_row_counts.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and get rows counts for all tables in all databases, +or only those matching given db / table / partition value regexes + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from hive_tables_row_counts import HiveTablesRowCounts +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + +class ImpalaTablesRowCounts(HiveTablesRowCounts): + + def __init__(self): + # Python 2.x + super(ImpalaTablesRowCounts, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesRowCounts().main() From 610d8568a7c1287b64bb98366a05b9ba460660fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:16:27 +0000 Subject: [PATCH 0745/2779] set verbose to only need one -v switch --- lib/hive_impala_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/hive_impala_cli.py b/lib/hive_impala_cli.py index d5f9a2a06..7515abddd 100755 --- a/lib/hive_impala_cli.py +++ b/lib/hive_impala_cli.py @@ -55,6 +55,7 @@ def __init__(self): self.kerberos = False self.krb5_service_name = self.default_service_name self.ssl = False + self.verbose_default = 1 #self.timeout_default = 86400 self.timeout_default = None if 'impala' in sys.argv[0]: From 3c8b4984f6172d4e76edf8efb2d0cfff6a0b2d1d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:30:06 +0000 Subject: [PATCH 0746/2779] refactored to subclass HiveImpalaCLI --- hive_foreach_table.py | 276 +++++++++++++++++------------------------- 1 file changed, 113 insertions(+), 163 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 90b294709..79833e7fc 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -16,7 +16,7 @@ """ -Tool to connect to a HiveServer2 / Impala node and execute a query for all tables in all databases, +Connect to HiveServer2 and execute a query for all tables in all databases, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: @@ -26,17 +26,12 @@ hive_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' hive_foreach_table.py --query 'ANALYZE TABLE {db}.{table} COMPUTE STATS' -impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' -impala_foreach_table.py --query 'COMPUTE STATS {table}' - or just for today's partition: hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS" -impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" - -Tested on CDH 5.10, Hive 1.1.0 and Impala 2.7.0 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see @@ -58,70 +53,49 @@ from __future__ import print_function from __future__ import unicode_literals -import argparse -import logging import os import re -import socket import sys import impala -from impala.dbapi import connect +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_regex + from hive_impala_cli import HiveImpalaCLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.0' - -logging.basicConfig() -log = logging.getLogger(os.path.basename(sys.argv[0])) - -def getenvs(keys, default=None): - for key in keys: - value = os.getenv(key) - if value: - return value - return default - -def parse_args(): - name = 'HiveServer2' - default_port = 10000 - default_service_name = 'hive' - host_envs = [ - 'HIVESERVER2_HOST', - 'HIVE_HOST', - 'HOST' - ] - port_envs = [ - 'HIVESERVER2_PORT', - 'HIVE_PORT', - 'PORT' - ] - - if 'impala' in sys.argv[0]: - name = 'Impala' - default_port = 21050 - default_service_name = 'impala' - host_envs = [ - 'IMPALA_HOST', - 'HOST' - ] - port_envs = [ - 'IMPALA_PORT', - 'PORT' - ] - parser = argparse.ArgumentParser(description="Executes a SQL statement for each matching {} table".format(name)) - parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ - help='{} host '.format(name) + \ - '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') - parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, default_port), - help='{} port (default: {}, '.format(name, default_port) + \ - ', $'.join(port_envs) + ')') - parser.add_argument('-q', '--query', required=True, help='Query or statement to execute for each table' + \ - ' (replaces {db} and {table} in the query string with each table and its database)') - parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') - parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') - parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') - parser.add_argument('-n', '--krb5-service-name', default=default_service_name, - help='Service principal (default: {})'.format(default_service_name)) - parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') +__version__ = '0.4.0' + + +class HiveForEachTable(HiveImpalaCLI): + + def __init__(self): + # Python 2.x + super(HiveForEachTable, self).__init__() + # Python 3.x + # super().__init__() + self.query = None + self.database = None + self.table = None + self.partition = None + self.ignore_errors = False + + def add_options(self): + super(HiveForEachTable, self).add_options() + self.add_opt('-q', '--query', help='Query or statement to execute for each table' + \ + ' (replaces {db} and {table} in the query string with each table and its database)') + self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)') + self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)') + #self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)') # # ignore tables that fail with errors like: # @@ -134,107 +108,83 @@ def parse_args(): # impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' # CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' # - parser.add_argument('-e', '--ignore-errors', action='store_true', help='Ignore errors and continue') - parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') - args = parser.parse_args() - - if args.verbose: - log.setLevel(logging.INFO) - if args.verbose > 1 or os.getenv('DEBUG'): - log.setLevel(logging.DEBUG) - - return args - -def connect_db(args, database): - auth_mechanism = None - if args.kerberos: - auth_mechanism = 'GSSAPI' - - log.info('connecting to %s:%s database %s', args.host, args.port, database) - return connect( - host=args.host, - port=args.port, - auth_mechanism=auth_mechanism, - use_ssl=args.ssl, - #user=user, - #password=password, - database=database, - kerberos_service_name=args.krb5_service_name - ) - -def main(): - args = parse_args() - - try: - database_regex = re.compile(args.database, re.I) - table_regex = re.compile(args.table, re.I) - except re.error as _: - log.error('error in provided regex: %s', _) - sys.exit(3) - - conn = connect_db(args, 'default') - - log.info('querying databases') - with conn.cursor() as db_cursor: - db_cursor.execute('show databases') - for db_row in db_cursor: - database = db_row[0] - if not database_regex.search(database): - log.debug("skipping database '%s', does not match regex '%s'", database, args.database) - continue - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, args.table) - continue - try: - query = args.query.format(db=database, table=table) - except KeyError as _: - if _ == 'db': - query = args.query.format(table=table) + self.add_opt('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue') + + def process_options(self): + super(HiveForEachTable, self).process_options() + self.query = self.get_opt('query') + if not self.query: + self.usage('query not defined') + self.database = self.get_opt('database') + self.table = self.get_opt('table') + #self.partition = self.get_opt('partition') + self.ignore_errors = self.get_opt('ignore_errors') + validate_regex(self.database, 'database') + validate_regex(self.table, 'table') + #validate_regex(self.partition, 'partition') + + def run(self): + database_regex = re.compile(self.database, re.I) + table_regex = re.compile(self.table, re.I) + #partition_regex = re.compile(self.partition, re.I) + conn = self.connect('default') + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, self.database) + continue + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: try: - execute(conn, database, table, query) - except Exception as _: - if args.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): continue raise - -def execute(conn, database, table, query): - try: - log.info(" %s.%s - running %s", database, table, query) - with conn.cursor() as query_cursor: - # doesn't support parameterized query quoting from dbapi spec - query_cursor.execute(query) - for result in query_cursor: - print('{db}.{table}\t{result}'.format(db=database, table=table, \ - result='\t'.join([str(_) for _ in result]))) - #except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: - # log.error(_) - except impala.error.ProgrammingError as _: - log.error(_) - # COMPUTE STATS returns no results - if 'Trying to fetch results on an operation with no results' not in str(_): - raise + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + try: + query = self.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = self.query.format(table=table) + try: + self.execute(conn, database, table, query) + except Exception as _: + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise + + @staticmethod + def execute(conn, database, table, query): + try: + log.info(" %s.%s - running %s", database, table, query) + with conn.cursor() as query_cursor: + # doesn't support parameterized query quoting from dbapi spec + query_cursor.execute(query) + for result in query_cursor: + print('{db}.{table}\t{result}'.format(db=database, table=table, \ + result='\t'.join([str(_) for _ in result]))) + #except (impala.error.OperationalError, impala.error.HiveServer2Error) as _: + # log.error(_) + except impala.error.ProgrammingError as _: + log.error(_) + # COMPUTE STATS returns no results + if 'Trying to fetch results on an operation with no results' not in str(_): + raise if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print("Control-C", file=sys.stderr) + HiveForEachTable().main() From 2b285e8abf621de8e5f407e632f9eb4f407b78ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:30:16 +0000 Subject: [PATCH 0747/2779] subclassed HiveForEachTable --- impala_foreach_table.py | 92 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) mode change 120000 => 100755 impala_foreach_table.py diff --git a/impala_foreach_table.py b/impala_foreach_table.py deleted file mode 120000 index 44716e6dc..000000000 --- a/impala_foreach_table.py +++ /dev/null @@ -1 +0,0 @@ -hive_foreach_table.py \ No newline at end of file diff --git a/impala_foreach_table.py b/impala_foreach_table.py new file mode 100755 index 000000000..49c8e21e0 --- /dev/null +++ b/impala_foreach_table.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and execute a query for all tables in all databases, +or only those matching given db / table regexes + +Useful for getting row counts of all tables or analyzing tables: + +eg. + +impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}' +impala_foreach_table.py --query 'COMPUTE STATS {table}' + +or just for today's partition: + +impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" + + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaForEachTable(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(ImpalaForEachTable, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaForEachTable().main() From 15e1475f1c6ead07b2092b39e28e9ca2a233d328 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:54:52 +0000 Subject: [PATCH 0748/2779] added lib/__init__.py --- lib/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/__init__.py diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 000000000..e69de29bb From 79bd12ba42efc5e5202eac0f381bcdc9d22fbf6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:57:04 +0000 Subject: [PATCH 0749/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index efdf62143..1dbb3dff3 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -138,7 +138,7 @@ def run(self): table_cursor.execute('show tables') for table_row in table_cursor: table = table_row[0] - log.info('describing table %s', table) + log.info('describing table %s.%s', database, table) with conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From 54f50ff9aa37ac561457edd6f8f7c1332fbd78a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 16:58:28 +0000 Subject: [PATCH 0750/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 79833e7fc..074a2b55d 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.4.1' class HiveForEachTable(HiveImpalaCLI): @@ -83,6 +83,8 @@ def __init__(self): super(HiveForEachTable, self).__init__() # Python 3.x # super().__init__() + + # self.query can be pre-defined in which case subclassed programs won't expose --query option self.query = None self.database = None self.table = None @@ -91,8 +93,10 @@ def __init__(self): def add_options(self): super(HiveForEachTable, self).add_options() - self.add_opt('-q', '--query', help='Query or statement to execute for each table' + \ - ' (replaces {db} and {table} in the query string with each table and its database)') + # allow subclassing to pre-define query and not expose option in that case + if self.query is None: + self.add_opt('-q', '--query', help='Query or statement to execute for each table' + \ + ' (replaces {db} and {table} in the query string with each table and its database)') self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)') self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)') #self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)') @@ -112,7 +116,8 @@ def add_options(self): def process_options(self): super(HiveForEachTable, self).process_options() - self.query = self.get_opt('query') + if self.query is None: + self.query = self.get_opt('query') if not self.query: self.usage('query not defined') self.database = self.get_opt('database') From 399b6bd1933ac2d12bc85b090eef54d381e6a49e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:46:35 +0000 Subject: [PATCH 0751/2779] added hive_tables_null_columns.py --- hive_tables_null_columns.py | 134 ++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100755 hive_tables_null_columns.py diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py new file mode 100755 index 000000000..02f47f4ef --- /dev/null +++ b/hive_tables_null_columns.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and find tables with columns containing only NULLs +for all tables in all databases, or only those matching given db / table regexes + +Describes each table to construct a large query based off all detected columns, +then queries the columns for NULLs and prints the tables columns with all NULLs + +Useful for catching data problems either caused by data input or broken ETL processes + +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + +class HiveTablesNullColumns(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesNullColumns, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # constructed later dynamically per table + self.database = None + self.table = None + #self.partition = None + self.ignore_errors = False + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + sum_part = '' + columns = [] + log.info('describing table %s.%s', database, table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use {}'.format(database)) + column_cursor.execute('describe {}'.format(table)) + for column_row in column_cursor: + column = column_row[0] + #column_type = column_row[1] + columns.append(column) + sum_part = ', '.join( + ['IF(SUM(IF({col} IS NULL, 1, 0)) = COUNT(*), 1, 0) as {col}'.format(col=column) \ + for column in columns] + ) + query = "SELECT {sum_part} FROM {db}.{table} WHERE "\ + .format(sum_part=sum_part, db=database, table=table) + \ + " IS NULL OR ".join(columns) + " IS NULL" + self.check_table_for_nulls(conn, database, table, columns, query) + + @staticmethod + def check_table_for_nulls(conn, database, table, columns, query): + with conn.cursor() as table_cursor: + log.debug('executing query: %s', query) + table_cursor.execute(query) + cols_with_nulls = [] + for result in table_cursor: + # tuple of ints (0, 0, 0, .... N) - one per column + for index in range(len(list(result))): + col_result = result[index] + if col_result > 0: + cols_with_nulls.append(columns[index]) + num_cols = len(cols_with_nulls) + total_cols = len(columns) + if cols_with_nulls: + print('WARNING: {db}.{table} has {num}/{total} columns with only NULLs: {cols}'\ + .format(db=database, + table=table, + num=num_cols, + total=total_cols, + cols=', '.join(sorted(cols_with_nulls)) + ) + ) + else: + print('OK: {db}.{table} as {num}/{total} columns with only nulls'\ + .format(db=database, table=table, num=num_cols, total=total_cols)) + + +if __name__ == '__main__': + HiveTablesNullColumns().main() From 11584ab5578e0bc88ecde053d401e336cb0b7f1c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:46:39 +0000 Subject: [PATCH 0752/2779] added impala_tables_null_columns.py --- impala_tables_null_columns.py | 82 +++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100755 impala_tables_null_columns.py diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py new file mode 100755 index 000000000..96cc7a998 --- /dev/null +++ b/impala_tables_null_columns.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and find tables with columns containing only NULLs +for all tables in all databases, or only those matching given db / table regexes + +Describes each table to construct a large query based off all detected columns, +then queries the columns for NULLs and prints the tables columns with all NULLs + +Useful for catching data problems either caused by data input or broken ETL processes + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_null_columns import HiveTablesNullColumns +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesNullColumns(HiveTablesNullColumns): + + def __init__(self): + # Python 2.x + super(ImpalaTablesNullColumns, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesNullColumns().main() From 42505f841b87f092da43e8812606ea91420be5d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:49:25 +0000 Subject: [PATCH 0753/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c32604247..2b895d3c4 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterate all matching tables in all databases (by partition if available) and output TSV of table names and row counts + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - iterate all matching tables in all databases and outputs tables columns containing only NULLs (useful for catching data quality or ETL problems) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 9cabf0ce21e91e8e408daa085aa2c93d43b4f9d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:50:14 +0000 Subject: [PATCH 0754/2779] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 02f47f4ef..4da274b88 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -19,10 +19,10 @@ Connect to HiveServer2 and find tables with columns containing only NULLs for all tables in all databases, or only those matching given db / table regexes -Describes each table to construct a large query based off all detected columns, -then queries the columns for NULLs and prints the tables columns with all NULLs +Describes each table to construct a large query based on all detected columns, +then queries the columns for NULLs and prints the tables columns containing only NULLs -Useful for catching data problems either caused by data input or broken ETL processes +Useful for catching problems with data quality or broken ETL processes Tested on Hive 1.1.0 on CDH 5.10 with Kerberos From c99e17fbe443181e6c3c1b2e9034cc4e8d04af52 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:53:44 +0000 Subject: [PATCH 0755/2779] updated impala_tables_null_columns.py --- impala_tables_null_columns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index 96cc7a998..05c149456 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -19,10 +19,10 @@ Connect to an Impala daemon and find tables with columns containing only NULLs for all tables in all databases, or only those matching given db / table regexes -Describes each table to construct a large query based off all detected columns, -then queries the columns for NULLs and prints the tables columns with all NULLs +Describes each table, constructs a complex query to check each column individually for containing only NULLs, +and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns -Useful for catching data problems either caused by data input or broken ETL processes +Useful for catching problems with data quality or broken ETL processes Tested on Impala 2.7.0 on CDH 5.10 with Kerberos From 6e1c3d1fbf648515cc2caed71ffb79d19a805f04 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:53:48 +0000 Subject: [PATCH 0756/2779] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 4da274b88..7dab3781e 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -19,8 +19,8 @@ Connect to HiveServer2 and find tables with columns containing only NULLs for all tables in all databases, or only those matching given db / table regexes -Describes each table to construct a large query based on all detected columns, -then queries the columns for NULLs and prints the tables columns containing only NULLs +Describes each table, constructs a complex query to check each column individually for containing only NULLs, +and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns Useful for catching problems with data quality or broken ETL processes From d67733b4cbd17b1a91726c4cad54691010f1ef29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:54:17 +0000 Subject: [PATCH 0757/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2b895d3c4..142917fad 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,8 @@ Environment variables are supported for convenience and also to hide credentials - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterate all matching tables in all databases (by partition if available) and output TSV of table names and row counts - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - iterate all matching tables in all databases and outputs tables columns containing only NULLs (useful for catching data quality or ETL problems) + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterates all matching tables in all databases (by partition if available) and outputs TSV of table names and row counts + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - iterates all matching tables in all databases and outputs tables columns containing only NULLs (useful for catching data quality or ETL problems) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From a04139218f34b7ae2f26f7181b9b3163b591ad69 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:58:39 +0000 Subject: [PATCH 0758/2779] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index e17a88e63..99f2d5b5a 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -19,6 +19,8 @@ Connect to an Impala daemon and get rows counts for all tables in all databases, or only those matching given db / table / partition value regexes +Useful for reconciliations between clusters after migrations + Tested on Impala 2.7.0 on CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From f3b6279126fbc8ca014a22e9ae5eaf0c98b1e118 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 17:58:43 +0000 Subject: [PATCH 0759/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 08eefbfff..ee7e933f1 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -19,6 +19,8 @@ Connect to HiveServer2 and get rows counts for all tables in all databases, or only those matching given db / table / partition value regexes +Useful for reconciliations between clusters after migrations + Tested on Hive 1.1.0 CDH 5.10 with Kerberos Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 34431bfdc84ef95e3f25bd9fd409f3882f8cba7c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 18:00:36 +0000 Subject: [PATCH 0760/2779] updated README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 142917fad..b69840f56 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,9 @@ Environment variables are supported for convenience and also to hide credentials - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement for every Hive / Impala table, optionally filtering to only select databases/tables via regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - iterates all matching tables in all databases (by partition if available) and outputs TSV of table names and row counts - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - iterates all matching tables in all databases and outputs tables columns containing only NULLs (useful for catching data quality or ETL problems) + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table, optionally limiting via database / table name regex + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs TSV list of tables and their row counts, optionally filtering by database / table name regex (useful for reconciliation between cluster migrations) + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs list of tables containing NULLs columns, optionally filtering by database / table name regex (useful for catching data quality or ETL problems) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 67c2cdd75ff45d7af3fc75dbc97145d1e8d5288e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jan 2020 18:01:33 +0000 Subject: [PATCH 0761/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b69840f56..51a4c0b38 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,8 @@ Environment variables are supported for convenience and also to hide credentials - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table, optionally limiting via database / table name regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs TSV list of tables and their row counts, optionally filtering by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs list of tables containing NULLs columns, optionally filtering by database / table name regex (useful for catching data quality or ETL problems) + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts, can filter by database / table name regex (useful for reconciliation between cluster migrations) + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs, can filter by database / table name regex (useful for catching data quality or ETL problems) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 9dd8e14117f1180a0ac293c60245403fc452f006 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:37:55 +0000 Subject: [PATCH 0762/2779] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 7dab3781e..9b66f0489 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -22,7 +22,9 @@ Describes each table, constructs a complex query to check each column individually for containing only NULLs, and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns -Useful for catching problems with data quality or broken ETL processes +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo Tested on Hive 1.1.0 on CDH 5.10 with Kerberos From 5b66fa8dc48b0c20cdc95b5c303c86567684aac6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:38:00 +0000 Subject: [PATCH 0763/2779] updated impala_tables_null_columns.py --- impala_tables_null_columns.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index 05c149456..62e984e64 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -22,7 +22,9 @@ Describes each table, constructs a complex query to check each column individually for containing only NULLs, and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns -Useful for catching problems with data quality or broken ETL processes +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo Tested on Impala 2.7.0 on CDH 5.10 with Kerberos From 423ef5e98a444fde933d02a6bbc29d9ecc3a4b3f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:39:20 +0000 Subject: [PATCH 0764/2779] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 9b66f0489..ffb1a7dce 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -128,7 +128,7 @@ def check_table_for_nulls(conn, database, table, columns, query): ) ) else: - print('OK: {db}.{table} as {num}/{total} columns with only nulls'\ + print('OK: {db}.{table} has {num}/{total} columns with only nulls'\ .format(db=database, table=table, num=num_cols, total=total_cols)) From 34edb23d60dcb80e3b6e53fd2370b65976c4701a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:50:49 +0000 Subject: [PATCH 0765/2779] quoted all column, db and table names to avoid column names like 'role' breaking queries --- hive_tables_null_columns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index ffb1a7dce..985619ec1 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -65,7 +65,7 @@ __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.5.0' class HiveTablesNullColumns(HiveForEachTable): @@ -96,12 +96,12 @@ def execute(self, conn, database, table, query): #column_type = column_row[1] columns.append(column) sum_part = ', '.join( - ['IF(SUM(IF({col} IS NULL, 1, 0)) = COUNT(*), 1, 0) as {col}'.format(col=column) \ + ['IF(SUM(IF(`{col}` IS NULL, 1, 0)) = COUNT(*), 1, 0) as `{col}`'.format(col=column) \ for column in columns] ) - query = "SELECT {sum_part} FROM {db}.{table} WHERE "\ + query = "SELECT {sum_part} FROM `{db}`.`{table}` WHERE `"\ .format(sum_part=sum_part, db=database, table=table) + \ - " IS NULL OR ".join(columns) + " IS NULL" + "` IS NULL OR `".join(columns) + "` IS NULL" self.check_table_for_nulls(conn, database, table, columns, query) @staticmethod From ae109aad324c71567ffe1ecd83a8754d913f1aa0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 09:56:59 +0000 Subject: [PATCH 0766/2779] quoted all db, table and column references --- hive_foreach_table.py | 4 ++-- hive_schemas_csv.py | 8 ++++---- hive_tables_null_columns.py | 4 ++-- hive_tables_row_counts.py | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 074a2b55d..423c1a78b 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.1' +__version__ = '0.4.2' class HiveForEachTable(HiveImpalaCLI): @@ -146,7 +146,7 @@ def run(self): try: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) + table_cursor.execute('use `{}`'.format(database)) table_cursor.execute('show tables') except impala.error.HiveServer2Error as _: log.error(_) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 1dbb3dff3..da6e8e889 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -77,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class HiveSchemasCSV(HiveImpalaCLI): @@ -134,7 +134,7 @@ def run(self): with conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) + table_cursor.execute('use `{}`'.format(database)) table_cursor.execute('show tables') for table_row in table_cursor: table = table_row[0] @@ -143,8 +143,8 @@ def run(self): # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use {}'.format(database)) - column_cursor.execute('describe {}'.format(table)) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) for column_row in column_cursor: column = column_row[0] column_type = column_row[1] diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 985619ec1..c11716b75 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -89,8 +89,8 @@ def execute(self, conn, database, table, query): # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use {}'.format(database)) - column_cursor.execute('describe {}'.format(table)) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) for column_row in column_cursor: column = column_row[0] #column_type = column_row[1] diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index ee7e933f1..36fa80811 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -125,7 +125,7 @@ def run(self): try: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use {}'.format(database)) + table_cursor.execute('use `{}`'.format(database)) table_cursor.execute('show tables') except impala.error.HiveServer2Error as _: log.error(_) @@ -152,9 +152,9 @@ def get_row_counts(self, conn, database, table, partition_regex): log.info("getting partitions for database '%s' table '%s'", database, table) with conn.cursor() as partition_cursor: # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('use {db}'.format(db=database)) + partition_cursor.execute('use `{db}`'.format(db=database)) try: - partition_cursor.execute('show partitions {table}'.format(table=table)) + partition_cursor.execute('show partitions `{table}`'.format(table=table)) for partitions_row in partition_cursor: partition_key = partitions_row[0] partition_value = partitions_row[1] @@ -168,7 +168,7 @@ def get_row_counts(self, conn, database, table, partition_regex): self.partition) continue # doesn't support parameterized query quoting from dbapi spec - partition_cursor.execute('SELECT COUNT(*) FROM {db}.{table} WHERE {key}={value}'\ + partition_cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}` WHERE `{key}`={value}'\ .format(db=database, table=table, key=partition_key, value=partition_value)) for result in partition_cursor: row_count = result[0] @@ -187,9 +187,9 @@ def get_row_counts(self, conn, database, table, partition_regex): log.info("no partitions found for database '%s' table '%s', getting row counts for whole table", database, table) with conn.cursor() as table_cursor: - log.info("running SELECT COUNT(*) FROM %s.%s", database, table) + log.info("running SELECT COUNT(*) FROM `%s`.`%s`", database, table) # doesn't support parameterized query quoting from dbapi spec - table_cursor.execute('SELECT COUNT(*) FROM {db}.{table}'.format(db=database, table=table)) + table_cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}`'.format(db=database, table=table)) for result in table_cursor: row_count = result[0] print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count)) From 5ba64d63be973a2460f452a2c34835f364423fac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:15:48 +0000 Subject: [PATCH 0767/2779] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index c11716b75..316d5336f 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -24,7 +24,7 @@ Useful for catching problems with data quality or subtle ETL bugs -Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo +Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo Tested on Hive 1.1.0 on CDH 5.10 with Kerberos From 580f749e1b6f0ad0ee8c3513fc32b8b84e620f9f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:15:59 +0000 Subject: [PATCH 0768/2779] updated impala_tables_null_columns.py --- impala_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index 62e984e64..bd2eb0e61 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -24,7 +24,7 @@ Useful for catching problems with data quality or subtle ETL bugs -Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo +Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo Tested on Impala 2.7.0 on CDH 5.10 with Kerberos From a22fbf1a5deea460134d00c9b28b93329b96f5e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:19:48 +0000 Subject: [PATCH 0769/2779] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 316d5336f..0c72c07aa 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -74,7 +74,7 @@ def __init__(self): super(HiveTablesNullColumns, self).__init__() # Python 3.x # super().__init__() - self.query = 'placeholder' # constructed later dynamically per table + self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option self.database = None self.table = None #self.partition = None From 8e41d7cade898da4a8fee457f66156f64c7bf000 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:31:45 +0000 Subject: [PATCH 0770/2779] added hive_tables_null_rows.py --- hive_tables_null_rows.py | 106 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 hive_tables_null_rows.py diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py new file mode 100755 index 000000000..60c43e79c --- /dev/null +++ b/hive_tables_null_rows.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and count number of rows with only NULLs in all columns +for each table in each database, or only those matching given db / table regexes + +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo + +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + +class HiveTablesNullRows(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesNullRows, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option + self.database = None + self.table = None + #self.partition = None + self.ignore_errors = False + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + columns = [] + log.info('describing table %s.%s', database, table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) + for column_row in column_cursor: + column = column_row[0] + #column_type = column_row[1] + columns.append(column) + query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL AND `".join(columns) + "` IS NULL" + with conn.cursor() as table_cursor: + log.debug('executing query: %s', query) + table_cursor.execute(query) + for result in table_cursor: + count = result[0] + print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) + + +if __name__ == '__main__': + HiveTablesNullRows().main() From 1cdb73bcf5d52f08906755c20157fb653d0bc5d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:31:52 +0000 Subject: [PATCH 0771/2779] added impala_tables_null_rows.py --- impala_tables_null_rows.py | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 impala_tables_null_rows.py diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py new file mode 100755 index 000000000..3da2b5a2a --- /dev/null +++ b/impala_tables_null_rows.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and count number of rows with only NULLs in all columns +for each tables in each database, or only those matching given db / table regexes + +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_null_rows import HiveTablesNullRows +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesNullRows(HiveTablesNullRows): + + def __init__(self): + # Python 2.x + super(ImpalaTablesNullRows, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesNullRows().main() From e9178a0cff24faf15ddb44dfdb18fd46dff5f610 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:34:06 +0000 Subject: [PATCH 0772/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 51a4c0b38..9989973f9 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,8 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table, optionally limiting via database / table name regex - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts, can filter by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs, can filter by database / table name regex (useful for catching data quality or ETL problems) + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Optionally filter by database / table name regex (useful for catching data quality or subtle ETL bugs) + - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Optionally filter by database / table name regex (useful for catching data quality or subtle ETL bugs) - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 480120392cd82109d622f637034e05a3a0047ed4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:42:54 +0000 Subject: [PATCH 0773/2779] added hive_tables_row_counts_any_nulls.py --- hive_tables_row_counts_any_nulls.py | 106 ++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 hive_tables_row_counts_any_nulls.py diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py new file mode 100755 index 000000000..72ea0c67c --- /dev/null +++ b/hive_tables_row_counts_any_nulls.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and count number of rows with NULL in any column +for each table in each database, or only those matching given db / table regexes + +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo + +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + +class HiveTablesRowsWithNulls(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesRowsWithNulls, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option + self.database = None + self.table = None + #self.partition = None + self.ignore_errors = False + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + columns = [] + log.info('describing table %s.%s', database, table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) + for column_row in column_cursor: + column = column_row[0] + #column_type = column_row[1] + columns.append(column) + query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL OR `".join(columns) + "` IS NULL" + with conn.cursor() as table_cursor: + log.debug('executing query: %s', query) + table_cursor.execute(query) + for result in table_cursor: + count = result[0] + print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) + + +if __name__ == '__main__': + HiveTablesRowsWithNulls().main() From 23f5d169fdf0a1f7e51e8436105d7297e3538167 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:42:59 +0000 Subject: [PATCH 0774/2779] added impala_tables_row_counts_any_nulls.py --- impala_tables_row_counts_any_nulls.py | 81 +++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 impala_tables_row_counts_any_nulls.py diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py new file mode 100755 index 000000000..5cfbc009d --- /dev/null +++ b/impala_tables_row_counts_any_nulls.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and count number of rows with NULL in any column +for each table in each database, or only those matching given db / table regexes + +Useful for catching problems with data quality or subtle ETL bugs + +Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_row_counts_any_nulls import HiveTablesRowsWithNulls +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesRowsWithNulls(HiveTablesRowsWithNulls): + + def __init__(self): + # Python 2.x + super(ImpalaTablesRowsWithNulls, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesRowsWithNulls().main() From 89b24267b6f1c0df963db413763d26402137c929 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:45:57 +0000 Subject: [PATCH 0775/2779] updated README.md --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9989973f9..44d1ffcc2 100644 --- a/README.md +++ b/README.md @@ -113,10 +113,11 @@ Environment variables are supported for convenience and also to hide credentials - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table, optionally limiting via database / table name regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts, can filter by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Optionally filter by database / table name regex (useful for catching data quality or subtle ETL bugs) - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Optionally filter by database / table name regex (useful for catching data quality or subtle ETL bugs) + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table. Optionally filter by database / table name regex + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Optionally filter by database / table name regex (useful for reconciliation between cluster migrations) + - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Optionally filter by database / table name regex (useful for reconciliation between cluster migrations) + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality or subtle ETL bugs. Optionally filter by database / table name regex + - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality or subtle ETL bugs. Optionally filter by database / table name regex - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From 857961a718a7c97cb7e4397a9e6b6e22bca054c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:49:43 +0000 Subject: [PATCH 0776/2779] updated README.md --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 44d1ffcc2..478a95a70 100644 --- a/README.md +++ b/README.md @@ -113,11 +113,12 @@ Environment variables are supported for convenience and also to hide credentials - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table. Optionally filter by database / table name regex - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Optionally filter by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Optionally filter by database / table name regex (useful for reconciliation between cluster migrations) - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality or subtle ETL bugs. Optionally filter by database / table name regex - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality or subtle ETL bugs. Optionally filter by database / table name regex + The following programs can all optionally filter by database / table name regex: + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table + - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations + - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs + - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs + - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From a72b47f8ae4ef9baf3af40eaf29d2e6c74867b14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 10:50:03 +0000 Subject: [PATCH 0777/2779] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 478a95a70..c52a0d279 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,9 @@ Environment variables are supported for convenience and also to hide credentials - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + The following programs can all optionally filter by database / table name regex: + - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs From e6431a7d1126a0b5a0b994ea03e3ad7327fb1fc2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 14:23:09 +0000 Subject: [PATCH 0778/2779] unnested iterations to avoid long running query handles incurring 'impala.error.HiveServer2Error: Invalid query handle' --- hive_foreach_table.py | 70 ++++++++++++++++++++++----------------- hive_tables_row_counts.py | 64 +++++++++++++++++++---------------- 2 files changed, 75 insertions(+), 59 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 423c1a78b..9ff0750eb 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.2' +__version__ = '0.5.0' class HiveForEachTable(HiveImpalaCLI): @@ -134,6 +134,9 @@ def run(self): #partition_regex = re.compile(self.partition, re.I) conn = self.connect('default') log.info('querying databases') + # collecting in local list because long time iteration results in + # impala.error.HiveServer2Error: Invalid query handle + databases = [] with conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: @@ -141,36 +144,41 @@ def run(self): if not database_regex.search(database): log.debug("skipping database '%s', does not match regex '%s'", database, self.database) continue - log.info('querying tables for database %s', database) - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, self.table) - continue - try: - query = self.query.format(db=database, table=table) - except KeyError as _: - if _ == 'db': - query = self.query.format(table=table) - try: - self.execute(conn, database, table, query) - except Exception as _: - if self.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) - continue - raise + databases.append(database) + for database in databases: + tables = [] + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + tables.append(table) + for table in tables: + try: + query = self.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = self.query.format(table=table) + try: + self.execute(conn, database, table, query) + except Exception as _: + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise @staticmethod def execute(conn, database, table, query): diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 36fa80811..6e93c6a84 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -63,7 +63,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.6.0' class HiveTablesRowCounts(HiveImpalaCLI): @@ -113,6 +113,9 @@ def run(self): partition_regex = re.compile(self.partition, re.I) conn = self.connect('default') log.info('querying databases') + # collecting in local list because long time iteration results in + # impala.error.HiveServer2Error: Invalid query handle + databases = [] with conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: @@ -120,33 +123,38 @@ def run(self): if not database_regex.search(database): log.debug("skipping database '%s', does not match regex '%s'", database, self.database) continue - log.info('querying tables for database %s', database) - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, self.table) - continue - try: - self.get_row_counts(conn, database, table, partition_regex) - except Exception as _: - # invalid query handle and similar errors happen at higher level - # as they are not query specific, will not be caught here so still error out - if self.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) - continue - raise + databases.append(database) + for database in databases: + tables = [] + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + continue + raise + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + tables.append(table) + for table in tables: + try: + self.get_row_counts(conn, database, table, partition_regex) + except Exception as _: + # invalid query handle and similar errors happen at higher level + # as they are not query specific, will not be caught here so still error out + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise def get_row_counts(self, conn, database, table, partition_regex): log.info("getting partitions for database '%s' table '%s'", database, table) From c8a692c1cfe484df6b9ac8a2aa3e9599738cf89b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:40:46 +0000 Subject: [PATCH 0779/2779] updated hdfs_find_replication_factor_1.py --- hdfs_find_replication_factor_1.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py index fa80c3586..0a38a9316 100755 --- a/hdfs_find_replication_factor_1.py +++ b/hdfs_find_replication_factor_1.py @@ -123,8 +123,7 @@ def run(self): print('', file=sys.stderr) print(file_path) if self.replication_factor: - log.info('setting replication factor to {} on {}'\ - .format(self.replication_factor, file_path)) + log.info('setting replication factor to %s on %s', self.replication_factor, file_path) # returns a generator so must evaluate in order to actually execute # otherwise you find there is no effect on the replication factor for _ in client.setrep([file_path], self.replication_factor, recurse=False): From 64a013c7addb4707568165d511072344acf37157 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:45:11 +0000 Subject: [PATCH 0780/2779] added database + table counters --- hive_foreach_table.py | 79 ++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 9ff0750eb..ea10ed5a7 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class HiveForEachTable(HiveImpalaCLI): @@ -137,53 +137,62 @@ def run(self): # collecting in local list because long time iteration results in # impala.error.HiveServer2Error: Invalid query handle databases = [] + database_count = 0 with conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: database = db_row[0] + database_count += 1 if not database_regex.search(database): log.debug("skipping database '%s', does not match regex '%s'", database, self.database) continue databases.append(database) + log.info('%s/%s databases selected', len(databases), database_count) for database in databases: - tables = [] - log.info('querying tables for database %s', database) - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, self.table) - continue - tables.append(table) - for table in tables: - try: - query = self.query.format(db=database, table=table) - except KeyError as _: - if _ == 'db': - query = self.query.format(table=table) - try: - self.execute(conn, database, table, query) - except Exception as _: - if self.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) - continue - raise + self.process_database(conn, database, table_regex) + + def process_database(self, conn, database, table_regex): + tables = [] + table_count = 0 + log.info("querying tables for database '%s'", database) + with conn.cursor() as table_cursor: + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error('error querying tables for database %s: %s', database, _) + if 'AuthorizationException' in str(_): + return + raise + for table_row in table_cursor: + table = table_row[0] + table_count += 1 + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + tables.append(table) + log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database) + for table in tables: + try: + query = self.query.format(db=database, table=table) + except KeyError as _: + if _ == 'db': + query = self.query.format(table=table) + try: + self.execute(conn, database, table, query) + except Exception as _: + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise @staticmethod def execute(conn, database, table, query): try: - log.info(" %s.%s - running %s", database, table, query) + log.info(' %s.%s - running %s', database, table, query) with conn.cursor() as query_cursor: # doesn't support parameterized query quoting from dbapi spec query_cursor.execute(query) From b8895386d05dd0cea7ea26d440654447e5dedfdc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:45:16 +0000 Subject: [PATCH 0781/2779] added database + table counters --- hive_schemas_csv.py | 109 ++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 33 deletions(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index da6e8e889..5fc218aa3 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -61,6 +61,7 @@ import csv import os import sys +import impala srcdir = os.path.abspath(os.path.dirname(__file__)) pylib = os.path.join(srcdir, 'pylib') lib = os.path.join(srcdir, 'lib') @@ -90,6 +91,11 @@ def __init__(self): self.delimiter = None self.quotechar = None self.escapechar = None + self.table_count = 0 + self.column_count = 0 + self.ignore_errors = False + self.csv_writer = None + self.conn = None def add_options(self): super(HiveSchemasCSV, self).add_options() @@ -101,57 +107,94 @@ def add_options(self): self.add_opt('-Q', '--quotechar', default='"', type=str, help='Generate quoted CSV (recommended, default is double quote \'"\')') self.add_opt('-E', '--escapechar', help='Escape char if needed') +# +# ignore tables that fail with errors like: +# +# Hive (CDH has MR, no tez): +# +# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long +# +# Impala: +# +# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table '
' +# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table '
' +# + self.add_opt('-e', '--ignore-errors', action='store_true', + help='Ignore individual table schema listing errors (Impala often has table metadata errors)') def process_options(self): super(HiveSchemasCSV, self).process_options() self.delimiter = self.get_opt('delimiter') self.quotechar = self.get_opt('quotechar') self.escapechar = self.get_opt('escapechar') + self.ignore_errors = self.get_opt('ignore_errors') def run(self): - conn = self.connect('default') + self.conn = self.connect('default') quoting = csv.QUOTE_ALL if self.quotechar == '': quoting = csv.QUOTE_NONE fieldnames = ['database', 'table', 'column', 'type'] - csv_writer = csv.DictWriter(sys.stdout, - delimiter=self.delimiter, - quotechar=self.quotechar, - escapechar=self.escapechar, - quoting=quoting, - fieldnames=fieldnames) - csv_writer.writeheader() + self.csv_writer = csv.DictWriter(sys.stdout, + delimiter=self.delimiter, + quotechar=self.quotechar, + escapechar=self.escapechar, + quoting=quoting, + fieldnames=fieldnames) + self.csv_writer.writeheader() log.info('querying databases') - with conn.cursor() as db_cursor: + databases = [] + database_count = 0 + with self.conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: database = db_row[0] - log.info('querying tables for database %s', database) - #db_conn = connect_db(args, database) - #with db_conn.cursor() as table_cursor: - with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - for table_row in table_cursor: - table = table_row[0] - log.info('describing table %s.%s', database, table) - with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec - #column_cursor.execute('use %(database)s', {'database': database}) - #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use `{}`'.format(database)) - column_cursor.execute('describe `{}`'.format(table)) - for column_row in column_cursor: - column = column_row[0] - column_type = column_row[1] - csv_writer.writerow({'database': database, - 'table': table, - 'column': column, - 'type': column_type}) + database_count += 1 + databases.append(database) + log.info('found %s databases', database_count) + for database in databases: + self.process_database(database) + log.info('databases: %s, tables: %s, columns: %s', database_count, self.table_count, self.column_count) + + def process_database(self, database): + log.info('querying tables for database %s', database) + tables = [] + with self.conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + self.table_count += 1 + tables.append(table) + for table in tables: + try: + self.process_table(database, table) + except impala.error.HiveServer2Error as _: + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise + + def process_table(self, database, table): + log.info('describing table %s.%s', database, table) + with self.conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use `{}`'.format(database)) + column_cursor.execute('describe `{}`'.format(table)) + for column_row in column_cursor: + column = column_row[0] + column_type = column_row[1] + self.column_count += 1 + self.csv_writer.writerow({'database': database, + 'table': table, + 'column': column, + 'type': column_type}) if __name__ == '__main__': From c066d6c251790e6de98e1e942709b07667eb9011 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:45:22 +0000 Subject: [PATCH 0782/2779] added database + table counters --- hive_tables_row_counts.py | 71 ++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 6e93c6a84..b9b06a0e8 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -63,7 +63,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.0' +__version__ = '0.6.1' class HiveTablesRowCounts(HiveImpalaCLI): @@ -116,45 +116,54 @@ def run(self): # collecting in local list because long time iteration results in # impala.error.HiveServer2Error: Invalid query handle databases = [] + database_count = 0 with conn.cursor() as db_cursor: db_cursor.execute('show databases') for db_row in db_cursor: database = db_row[0] + database_count += 1 if not database_regex.search(database): log.debug("skipping database '%s', does not match regex '%s'", database, self.database) continue databases.append(database) + log.info('%s/%s databases selected', len(databases), database_count) for database in databases: - tables = [] - log.info('querying tables for database %s', database) - with conn.cursor() as table_cursor: - try: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute('show tables') - except impala.error.HiveServer2Error as _: - log.error(_) - if 'AuthorizationException' in str(_): - continue - raise - for table_row in table_cursor: - table = table_row[0] - if not table_regex.search(table): - log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ - database, table, self.table) - continue - tables.append(table) - for table in tables: - try: - self.get_row_counts(conn, database, table, partition_regex) - except Exception as _: - # invalid query handle and similar errors happen at higher level - # as they are not query specific, will not be caught here so still error out - if self.ignore_errors: - log.error("database '%s' table '%s': %s", database, table, _) - continue - raise + self.process_database(conn, database, table_regex, partition_regex) + + def process_database(self, conn, database, table_regex, partition_regex): + tables = [] + table_count = 0 + log.info('querying tables for database %s', database) + with conn.cursor() as table_cursor: + try: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute('show tables') + except impala.error.HiveServer2Error as _: + log.error(_) + if 'AuthorizationException' in str(_): + return + raise + for table_row in table_cursor: + table = table_row[0] + table_count += 1 + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, self.table) + continue + tables.append(table) + log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database) + for table in tables: + try: + self.get_row_counts(conn, database, table, partition_regex) + except Exception as _: + # invalid query handle and similar errors happen at higher level + # as they are not query specific, will not be caught here so still error out + if self.ignore_errors: + log.error("database '%s' table '%s': %s", database, table, _) + continue + raise def get_row_counts(self, conn, database, table, partition_regex): log.info("getting partitions for database '%s' table '%s'", database, table) From cbe5a865df624e031773f3f133db89b82e9f4fca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:47:41 +0000 Subject: [PATCH 0783/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 5fc218aa3..c47fbad13 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -170,6 +170,7 @@ def process_database(self, database): table = table_row[0] self.table_count += 1 tables.append(table) + log.info("found %s tables in database '%s'", len(tables), database) for table in tables: try: self.process_table(database, table) From 0a4d6cfd3c015b4bb624e6f7adb856601023f3aa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:49:12 +0000 Subject: [PATCH 0784/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index c47fbad13..eddef782c 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -188,14 +188,17 @@ def process_table(self, database, table): #column_cursor.execute('describe %(table)s', {'table': table}) column_cursor.execute('use `{}`'.format(database)) column_cursor.execute('describe `{}`'.format(table)) + column_count = 0 for column_row in column_cursor: column = column_row[0] column_type = column_row[1] - self.column_count += 1 + column_count += 1 self.csv_writer.writerow({'database': database, 'table': table, 'column': column, 'type': column_type}) + log.info("found %s columns in table '%s.%s'", column_count, database, table) + self.column_count += column_count if __name__ == '__main__': From fa5db3aef2ace8c3456615be8f7b18368ef046f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:52:01 +0000 Subject: [PATCH 0785/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index ea10ed5a7..245e9bdf5 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -90,6 +90,7 @@ def __init__(self): self.table = None self.partition = None self.ignore_errors = False + self.table_count = 0 def add_options(self): super(HiveForEachTable, self).add_options() @@ -150,6 +151,7 @@ def run(self): log.info('%s/%s databases selected', len(databases), database_count) for database in databases: self.process_database(conn, database, table_regex) + log.info('processed %s databases, %s tables', database_count, self.table_count) def process_database(self, conn, database, table_regex): tables = [] @@ -183,6 +185,7 @@ def process_database(self, conn, database, table_regex): query = self.query.format(table=table) try: self.execute(conn, database, table, query) + self.table_count += 1 except Exception as _: if self.ignore_errors: log.error("database '%s' table '%s': %s", database, table, _) From ad7b1ab6843c1debaadfa62f4ad7fbd88dd13bb2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:55:57 +0000 Subject: [PATCH 0786/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index b9b06a0e8..52726e063 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -77,6 +77,7 @@ def __init__(self): self.table = None self.partition = None self.ignore_errors = False + self.table_count = 0 def add_options(self): super(HiveTablesRowCounts, self).add_options() @@ -129,6 +130,7 @@ def run(self): log.info('%s/%s databases selected', len(databases), database_count) for database in databases: self.process_database(conn, database, table_regex, partition_regex) + log.info('processed %s databases, %s tables', database_count, self.table_count) def process_database(self, conn, database, table_regex, partition_regex): tables = [] @@ -157,6 +159,7 @@ def process_database(self, conn, database, table_regex, partition_regex): for table in tables: try: self.get_row_counts(conn, database, table, partition_regex) + self.table_count += 1 except Exception as _: # invalid query handle and similar errors happen at higher level # as they are not query specific, will not be caught here so still error out From 7d91b62a2b3f2138512f76fb7e182ed8da91f871 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:57:29 +0000 Subject: [PATCH 0787/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index eddef782c..9a687152b 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -159,7 +159,7 @@ def run(self): log.info('databases: %s, tables: %s, columns: %s', database_count, self.table_count, self.column_count) def process_database(self, database): - log.info('querying tables for database %s', database) + log.info("querying tables for database '%s'", database) tables = [] with self.conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec From 8b031b983a7b99b32e47e57ce60ca90f97659e85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:57:47 +0000 Subject: [PATCH 0788/2779] updated hive_tables_row_counts.py --- hive_tables_row_counts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index 52726e063..d62ecf77c 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -135,7 +135,7 @@ def run(self): def process_database(self, conn, database, table_regex, partition_regex): tables = [] table_count = 0 - log.info('querying tables for database %s', database) + log.info("querying tables for database '%s'", database) with conn.cursor() as table_cursor: try: # doesn't support parameterized query quoting from dbapi spec From 8f20568c52f651e5b72fcb8657ef38901edb2c5d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:58:10 +0000 Subject: [PATCH 0789/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 245e9bdf5..0f783fce4 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -164,7 +164,7 @@ def process_database(self, conn, database, table_regex): table_cursor.execute('use `{}`'.format(database)) table_cursor.execute('show tables') except impala.error.HiveServer2Error as _: - log.error('error querying tables for database %s: %s', database, _) + log.error("error querying tables for database '%s': %s", database, _) if 'AuthorizationException' in str(_): return raise From b71bf376bcd777c8428d86d8a53fd40e64e0d5f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 16:59:44 +0000 Subject: [PATCH 0790/2779] updated hive_schemas_csv.py --- hive_schemas_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 9a687152b..9f4ee016a 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -181,7 +181,7 @@ def process_database(self, database): raise def process_table(self, database, table): - log.info('describing table %s.%s', database, table) + log.info("describing table '%s.%s'", database, table) with self.conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From bc0597e68ecfc144b7f52c6d27854ce320f28a48 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:00:14 +0000 Subject: [PATCH 0791/2779] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 0c72c07aa..3f16e9f5c 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -84,7 +84,7 @@ def __init__(self): def execute(self, conn, database, table, query): sum_part = '' columns = [] - log.info('describing table %s.%s', database, table) + log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From 88abb33c82c87bd6ede9f3c520037c85fafb7f29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:00:23 +0000 Subject: [PATCH 0792/2779] updated hive_tables_null_rows.py --- hive_tables_null_rows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 60c43e79c..698b72a78 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -80,7 +80,7 @@ def __init__(self): # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): columns = [] - log.info('describing table %s.%s', database, table) + log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From 7fccea2201627e3727e5f4791464179fe1c0fff0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:00:37 +0000 Subject: [PATCH 0793/2779] updated hive_tables_row_counts_any_nulls.py --- hive_tables_row_counts_any_nulls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 72ea0c67c..21e248f2b 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -80,7 +80,7 @@ def __init__(self): # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): columns = [] - log.info('describing table %s.%s', database, table) + log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: # doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) From e26f222c51c183bd471da581c5fbb0a34b3327fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:08:03 +0000 Subject: [PATCH 0794/2779] added hive_tables_list.py --- hive_tables_list.py | 80 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 hive_tables_list.py diff --git a/hive_tables_list.py b/hive_tables_list.py new file mode 100755 index 000000000..7416dd298 --- /dev/null +++ b/hive_tables_list.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and list all databases and tables + +TSV Output format: + +
+ + +Tested on Hive 1.1.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +class HiveTablesList(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesList, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # here merely to suppress --query CLI option + + def execute(self, conn, database, table, query): + print('{}\t{}'.format(database, table)) + + +if __name__ == '__main__': + HiveTablesList().main() From f21ee01adb5973a033b37fa97bb790adbcfa3b4d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:08:11 +0000 Subject: [PATCH 0795/2779] added impala_tables_list.py --- impala_tables_list.py | 81 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 impala_tables_list.py diff --git a/impala_tables_list.py b/impala_tables_list.py new file mode 100755 index 000000000..caddf99ec --- /dev/null +++ b/impala_tables_list.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and list all databases and tables + +TSV Output format: + +
+ + +Tested on Impala 2.7.0 on CDH 5.10 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_list import HiveTablesList +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class ImpalaTablesList(HiveTablesList): + + def __init__(self): + # Python 2.x + super(ImpalaTablesList, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesList().main() From f9588522a1e558a3ab1f7fb88f958818cacf0569 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 17:11:48 +0000 Subject: [PATCH 0796/2779] updated tested versions --- hive_foreach_table.py | 2 +- hive_schemas_csv.py | 2 +- hive_tables_list.py | 2 +- hive_tables_null_columns.py | 2 +- hive_tables_null_rows.py | 2 +- hive_tables_row_counts.py | 2 +- hive_tables_row_counts_any_nulls.py | 2 +- impala_foreach_table.py | 2 +- impala_schemas_csv.py | 2 +- impala_tables_list.py | 2 +- impala_tables_null_columns.py | 2 +- impala_tables_null_rows.py | 2 +- impala_tables_row_counts.py | 2 +- impala_tables_row_counts_any_nulls.py | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 0f783fce4..edb15e434 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -31,7 +31,7 @@ hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS" -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 9f4ee016a..5daf009d6 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -36,7 +36,7 @@ if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will raise a traceback to tell you to set one (eg. --escapechar='\\') -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_list.py b/hive_tables_list.py index 7416dd298..b0c4f4702 100755 --- a/hive_tables_list.py +++ b/hive_tables_list.py @@ -23,7 +23,7 @@
-Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 3f16e9f5c..a78e049f4 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -26,7 +26,7 @@ Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 698b72a78..78c42d15e 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -23,7 +23,7 @@ Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index d62ecf77c..b37059c9b 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -21,7 +21,7 @@ Useful for reconciliations between clusters after migrations -Tested on Hive 1.1.0 CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 21e248f2b..0f7a5a94a 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -23,7 +23,7 @@ Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo -Tested on Hive 1.1.0 on CDH 5.10 with Kerberos +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 49c8e21e0..7b35b0174 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -31,7 +31,7 @@ impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))" -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py index 17ce8987b..d6555b542 100755 --- a/impala_schemas_csv.py +++ b/impala_schemas_csv.py @@ -36,7 +36,7 @@ if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will raise a traceback to tell you to set one (eg. --escapechar='\\') -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_list.py b/impala_tables_list.py index caddf99ec..feb4e73ca 100755 --- a/impala_tables_list.py +++ b/impala_tables_list.py @@ -23,7 +23,7 @@
-Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index bd2eb0e61..ad968fbc1 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -26,7 +26,7 @@ Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py index 3da2b5a2a..c70484540 100755 --- a/impala_tables_null_rows.py +++ b/impala_tables_null_rows.py @@ -23,7 +23,7 @@ Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 99f2d5b5a..47c104789 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -21,7 +21,7 @@ Useful for reconciliations between clusters after migrations -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py index 5cfbc009d..ab6a6c638 100755 --- a/impala_tables_row_counts_any_nulls.py +++ b/impala_tables_row_counts_any_nulls.py @@ -23,7 +23,7 @@ Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo -Tested on Impala 2.7.0 on CDH 5.10 with Kerberos +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From f179178e1d2840e4e47229f1474af83936d2692c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 18:09:34 +0000 Subject: [PATCH 0797/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 7335c0168..966142672 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -488,6 +488,9 @@ dest[140]='ssh -i myKey -N -L 8888::8888 @' src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db/
/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" +src[142]="ERROR: AnalysisException: Failed to load metadata for table: 'myCustomerTable2'" +dest[142]="ERROR: AnalysisException: Failed to load metadata for table: '
'" + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" From 1da475945ce57620c5d65410e5617b73ffa00643 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 28 Jan 2020 18:09:44 +0000 Subject: [PATCH 0798/2779] updated anonymize.py --- anonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonymize.py b/anonymize.py index 106f73952..94beaf3f6 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.9' +__version__ = '0.10.10' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -319,7 +319,7 @@ def __init__(self): .format(arg_sep=arg_sep, id_or_name=id_or_name, switch_prefix=switch_prefix), - 'db4': r'(\s(?:in|of)\s+(column|table|database|schema)\s+[\'"])[^\'"]+', + 'db4': r'(\s(?:in|of|for)\s+(column|table|database|schema)[\s:]+[\'"])[^\'"]+', 'db5': r'/+user/+hive/+warehouse/+([A-Za-z0-9_-]+/+)*[A-Za-z0-9_-]+.db/+[A-Za-z0-9_-]+', 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex), 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\ From 9c693e1c5141aa3ac4809ebc20edc95f43ad49d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Jan 2020 14:25:40 +0000 Subject: [PATCH 0799/2779] added quoting and connect to each db --- hive_foreach_table.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index edb15e434..4ef224290 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -73,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.1' +__version__ = '0.5.2' class HiveForEachTable(HiveImpalaCLI): @@ -150,13 +150,14 @@ def run(self): databases.append(database) log.info('%s/%s databases selected', len(databases), database_count) for database in databases: - self.process_database(conn, database, table_regex) + self.process_database(database, table_regex) log.info('processed %s databases, %s tables', database_count, self.table_count) - def process_database(self, conn, database, table_regex): + def process_database(self, database, table_regex): tables = [] table_count = 0 log.info("querying tables for database '%s'", database) + conn = self.connect(database) with conn.cursor() as table_cursor: try: # doesn't support parameterized query quoting from dbapi spec @@ -179,10 +180,13 @@ def process_database(self, conn, database, table_regex): log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database) for table in tables: try: - query = self.query.format(db=database, table=table) + query = self.query.format(db='`{}`'.format(database), + table='`{}`'.format(table)) except KeyError as _: if _ == 'db': - query = self.query.format(table=table) + query = self.query.format(table='`{}`'.format(table)) + else: + raise try: self.execute(conn, database, table, query) self.table_count += 1 From 0e8f4c6615153c8f660c1f3aa19710a95c5a3d7c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 14:50:32 +0000 Subject: [PATCH 0800/2779] added hive_tables_locations.py --- hive_tables_locations.py | 90 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100755 hive_tables_locations.py diff --git a/hive_tables_locations.py b/hive_tables_locations.py new file mode 100755 index 000000000..cadccf9e2 --- /dev/null +++ b/hive_tables_locations.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and list the locations of all tables in all databases, +or only those matching given db / table regexes + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveTablesLocations(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesLocations, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'describe formatted {table}' + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + log.info("describing table '%s.%s'", database, table) + location = 'UNKNOWN' + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + #table_cursor.execute('describe %(table)s', {'table': table}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute(query.format(table=table)) + for row in table_cursor: + if 'Location' in row[0]: + location = row[1] + break + print('{db}.{table}\t{location}'.format(db=database, table=table, location=location)) + + +if __name__ == '__main__': + HiveTablesLocations().main() From c39c8f803aa3e3e18b2686dd771a56c155f434f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 14:50:37 +0000 Subject: [PATCH 0801/2779] added impala_tables_locations.py --- impala_tables_locations.py | 77 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 impala_tables_locations.py diff --git a/impala_tables_locations.py b/impala_tables_locations.py new file mode 100755 index 000000000..1182164c6 --- /dev/null +++ b/impala_tables_locations.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and list the locations of all tables in all databases, +or only those matching given db / table regexes + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_locations import HiveTablesLocations +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesLocations(HiveTablesLocations): + + def __init__(self): + # Python 2.x + super(ImpalaTablesLocations, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesLocations().main() From 4823788f6f482a20fbc9a6eb6ec80882990fdeb1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 14:58:40 +0000 Subject: [PATCH 0802/2779] added hive_tables_metadata.py --- hive_tables_metadata.py | 111 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100755 hive_tables_metadata.py diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py new file mode 100755 index 000000000..c2e20552c --- /dev/null +++ b/hive_tables_metadata.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') +for all tables in all databases, or only those matching given db / table regexes + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import re +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_regex + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveTablesMetadata(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesMetadata, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'describe formatted {table}' + self.field = None + + def add_options(self): + # Python 2.x + super(HiveTablesMetadata, self).add_options() + # Python 3.x + # super().__init__() + if self.field is None: + self.add_opt('-f', '--field', help='Table DDL metadata field to return for each table (required)') + + def process_options(self): + # Python 2.x + super(HiveTablesMetadata, self).process_options() + # Python 3.x + # super().__init__() + field = self.get_opt('field') + if not field: + self.usage('--field not specified') + validate_regex(field, 'field') + self.field = re.compile(field) + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + log.info("describing table '%s.%s'", database, table) + location = 'UNKNOWN' + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + #table_cursor.execute('describe %(table)s', {'table': table}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute(query.format(table=table)) + for row in table_cursor: + if self.field.search(row[0]): + location = row[1] + break + print('{db}.{table}\t{location}'.format(db=database, table=table, location=location)) + + +if __name__ == '__main__': + HiveTablesMetadata().main() From 16500c28e2f0954841bcd89ea9d478aa290d0c0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 14:58:43 +0000 Subject: [PATCH 0803/2779] added impala_tables_metadata.py --- impala_tables_metadata.py | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 impala_tables_metadata.py diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py new file mode 100755 index 000000000..8182f74e1 --- /dev/null +++ b/impala_tables_metadata.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and list the locations of all tables in all databases, +or only those matching given db / table regexes + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_metadata import HiveTablesMetadata +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesMetadata(HiveTablesMetadata): + + def __init__(self): + # Python 2.x + super(ImpalaTablesMetadata, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesMetadata().main() From 57cedbe51323f2623d1a6a7d465b812bf890f573 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 15:01:17 +0000 Subject: [PATCH 0804/2779] updated hive_tables_metadata.py --- hive_tables_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index c2e20552c..6de7888cb 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -84,7 +84,8 @@ def process_options(self): super(HiveTablesMetadata, self).process_options() # Python 3.x # super().__init__() - field = self.get_opt('field') + if self.field is None: + field = self.get_opt('field') if not field: self.usage('--field not specified') validate_regex(field, 'field') From ef6473e124eb5ed0e2ad09cf7e79573a7d1c71d2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 15:01:28 +0000 Subject: [PATCH 0805/2779] updated hive_tables_locations.py --- hive_tables_locations.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/hive_tables_locations.py b/hive_tables_locations.py index cadccf9e2..af17aaed8 100755 --- a/hive_tables_locations.py +++ b/hive_tables_locations.py @@ -48,8 +48,7 @@ sys.path.append(pylib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log - from hive_foreach_table import HiveForEachTable + from hive_tables_metadata import HiveTablesMetadata except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) print("Did you remember to build the project by running 'make'?", file=sys.stderr) @@ -60,30 +59,14 @@ __version__ = '0.5.0' -class HiveTablesLocations(HiveForEachTable): +class HiveTablesLocations(HiveTablesMetadata): def __init__(self): # Python 2.x super(HiveTablesLocations, self).__init__() # Python 3.x # super().__init__() - self.query = 'describe formatted {table}' - - # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): - log.info("describing table '%s.%s'", database, table) - location = 'UNKNOWN' - with conn.cursor() as table_cursor: - # doesn't support parameterized query quoting from dbapi spec - #table_cursor.execute('use %(database)s', {'database': database}) - #table_cursor.execute('describe %(table)s', {'table': table}) - table_cursor.execute('use `{}`'.format(database)) - table_cursor.execute(query.format(table=table)) - for row in table_cursor: - if 'Location' in row[0]: - location = row[1] - break - print('{db}.{table}\t{location}'.format(db=database, table=table, location=location)) + self.field = 'Location' if __name__ == '__main__': From 46d3954fc58d8f0324d0565b4bb5a5c2b5c81355 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 15:02:47 +0000 Subject: [PATCH 0806/2779] updated hive_tables_metadata.py --- hive_tables_metadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 6de7888cb..ccf66fc2a 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -85,11 +85,11 @@ def process_options(self): # Python 3.x # super().__init__() if self.field is None: - field = self.get_opt('field') - if not field: + self.field = self.get_opt('field') + if not self.field: self.usage('--field not specified') - validate_regex(field, 'field') - self.field = re.compile(field) + validate_regex(self.field, 'field') + self.field = re.compile(self.field) # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): From e48535cf3385a13cac643d15b3d7353c77b8c6db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 15:05:52 +0000 Subject: [PATCH 0807/2779] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c52a0d279..342ceb725 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,8 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs + - ```hive_tables_metadata.py``` / ```impala_tables_metadata.py``` - outputs for each table the matching regex metadata DDL property from describe table + - ```hive_tables_locations.py``` / ```impala_tables_locations.py``` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables From eeab4a654786f323a574f65107d48c4e5a36acd4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:38:20 +0000 Subject: [PATCH 0808/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1e33eaec4..dac327ef0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1e33eaec4117818ee32e3644e101a715f5a66de0 +Subproject commit dac327ef02f570cd08b426a4e9e90a97d0171357 From 23fd6c99c25b4c3f3711d57c4246053834cff9cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:38:21 +0000 Subject: [PATCH 0809/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4dcc62e62..ade8241d3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4dcc62e62cbc569a0422a1f57be20ceef32054d0 +Subproject commit ade8241d30c3f7049c65d6de6fb6df1dad273b83 From dee2eb59b10c94262a5332bbeba91f969c2cb8c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0810/2779] added .github/workflows/ci_alpine.yaml --- .github/workflows/ci_alpine.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/ci_alpine.yaml diff --git a/.github/workflows/ci_alpine.yaml b/.github/workflows/ci_alpine.yaml new file mode 100644 index 000000000..1a10eb7b2 --- /dev/null +++ b/.github/workflows/ci_alpine.yaml @@ -0,0 +1,44 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Alpine + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: harisekhon/alpine-dev + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: build & test + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools && + cd devops-python-tools && + make build test From 86d69a8673347235eb91cd99704ec44cca1fe921 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0811/2779] added .github/workflows/ci_centos.yaml --- .github/workflows/ci_centos.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/ci_centos.yaml diff --git a/.github/workflows/ci_centos.yaml b/.github/workflows/ci_centos.yaml new file mode 100644 index 000000000..4aba51533 --- /dev/null +++ b/.github/workflows/ci_centos.yaml @@ -0,0 +1,44 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI CentOS + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: harisekhon/centos-dev + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: build & test + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools && + cd devops-python-tools && + make build test From 05af7465ffca413357c213ecc554c89c73cca619 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0812/2779] added .github/workflows/ci_mac.yaml --- .github/workflows/ci_mac.yaml | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/ci_mac.yaml diff --git a/.github/workflows/ci_mac.yaml b/.github/workflows/ci_mac.yaml new file mode 100644 index 000000000..8994825ff --- /dev/null +++ b/.github/workflows/ci_mac.yaml @@ -0,0 +1,41 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:39:47 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Mac + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v1 + with: + path: ~/Library/Caches/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: build & test + run: make build test From a2592f847dfa586c03fba9d4c1489faf7c9511ba Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0813/2779] added .github/workflows/ci_python_2.7.yaml --- .github/workflows/ci_python_2.7.yaml | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/ci_python_2.7.yaml diff --git a/.github/workflows/ci_python_2.7.yaml b/.github/workflows/ci_python_2.7.yaml new file mode 100644 index 000000000..8a5fea1a4 --- /dev/null +++ b/.github/workflows/ci_python_2.7.yaml @@ -0,0 +1,49 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 2.7 + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [2.7] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: build & test + run: make build test From 7fed31e0b7e75dc9d6807b81176e0da2aef598a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0814/2779] added .github/workflows/ci_python_3.6.yaml --- .github/workflows/ci_python_3.6.yaml | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/ci_python_3.6.yaml diff --git a/.github/workflows/ci_python_3.6.yaml b/.github/workflows/ci_python_3.6.yaml new file mode 100644 index 000000000..2822288a0 --- /dev/null +++ b/.github/workflows/ci_python_3.6.yaml @@ -0,0 +1,49 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 3.6 + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [3.6] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: build & test + run: make build test From c5a578c33f5035f57870c6c8b7f476ebf2d3322e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:40:16 +0000 Subject: [PATCH 0815/2779] added .github/workflows/ci_ubuntu.yaml --- .github/workflows/ci_ubuntu.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/ci_ubuntu.yaml diff --git a/.github/workflows/ci_ubuntu.yaml b/.github/workflows/ci_ubuntu.yaml new file mode 100644 index 000000000..c48b25b1b --- /dev/null +++ b/.github/workflows/ci_ubuntu.yaml @@ -0,0 +1,44 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu + +env: + DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, ubuntu-16.04] + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: build & test + run: make build test From 62040b10672dac2d5c3bd641e7bd7996dfc55dca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 21:41:05 +0000 Subject: [PATCH 0816/2779] updated README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 342ceb725..c246cdde3 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,13 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) +[![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) +[![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) +[![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) +[![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) +[![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) +[![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) + ### AWS, Docker, Spark / PySpark, Hadoop, HBase, Hive, Impala, Pig, Ambari, IPython and Linux Tools ### A few of the Cloud, Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options. From b74d35865d966c482cc5532025229fe2d3d5329c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:15 +0000 Subject: [PATCH 0817/2779] updated ci_alpine.yaml --- .github/workflows/ci_alpine.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_alpine.yaml b/.github/workflows/ci_alpine.yaml index 1a10eb7b2..1b5a0f43c 100644 --- a/.github/workflows/ci_alpine.yaml +++ b/.github/workflows/ci_alpine.yaml @@ -13,8 +13,8 @@ name: CI Alpine -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From 72ed22931af6a916a1cb05a0e9cddd1f50e259f6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:25 +0000 Subject: [PATCH 0818/2779] updated ci_centos.yaml --- .github/workflows/ci_centos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_centos.yaml b/.github/workflows/ci_centos.yaml index 4aba51533..1d8071032 100644 --- a/.github/workflows/ci_centos.yaml +++ b/.github/workflows/ci_centos.yaml @@ -13,8 +13,8 @@ name: CI CentOS -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From 815089e0e4ec08d8033b2a2124b90094cb579a9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:31 +0000 Subject: [PATCH 0819/2779] updated ci_mac.yaml --- .github/workflows/ci_mac.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_mac.yaml b/.github/workflows/ci_mac.yaml index 8994825ff..36f71e43d 100644 --- a/.github/workflows/ci_mac.yaml +++ b/.github/workflows/ci_mac.yaml @@ -13,8 +13,8 @@ name: CI Mac -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From 00742a66a26ad17a87735160eef6417add5b974d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:37 +0000 Subject: [PATCH 0820/2779] updated ci_python_2.7.yaml --- .github/workflows/ci_python_2.7.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_python_2.7.yaml b/.github/workflows/ci_python_2.7.yaml index 8a5fea1a4..e9489be32 100644 --- a/.github/workflows/ci_python_2.7.yaml +++ b/.github/workflows/ci_python_2.7.yaml @@ -13,8 +13,8 @@ name: CI Python 2.7 -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From 9f2b8e30fde18b67e59d9b470a9db28d5868c047 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:43 +0000 Subject: [PATCH 0821/2779] updated ci_python_3.6.yaml --- .github/workflows/ci_python_3.6.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_python_3.6.yaml b/.github/workflows/ci_python_3.6.yaml index 2822288a0..16932c744 100644 --- a/.github/workflows/ci_python_3.6.yaml +++ b/.github/workflows/ci_python_3.6.yaml @@ -13,8 +13,8 @@ name: CI Python 3.6 -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From af6adf58c9be8206741013a65c37f054cde2d6ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 22:46:50 +0000 Subject: [PATCH 0822/2779] updated ci_ubuntu.yaml --- .github/workflows/ci_ubuntu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_ubuntu.yaml b/.github/workflows/ci_ubuntu.yaml index c48b25b1b..a919dbf9d 100644 --- a/.github/workflows/ci_ubuntu.yaml +++ b/.github/workflows/ci_ubuntu.yaml @@ -13,8 +13,8 @@ name: CI Ubuntu -env: - DEBUG: 1 +#env: +# DEBUG: 1 on: push: From f7234f960292b9d4e7c23f1f3835ab7117710c47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Feb 2020 23:37:08 +0000 Subject: [PATCH 0823/2779] updated all.sh --- tests/all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all.sh b/tests/all.sh index 18dddfc25..7aa1f8c75 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -31,7 +31,7 @@ section "Running PyTools ALL" cd "$srcdir/.."; bash-tools/check_all.sh -tests/test_yamllint.sh +#tests/test_yamllint.sh # do help afterwards for Spark to be downloaded, and then help will find and use downloaded spark for SPARK_HOME exit 0 From 489cf9095736001341a4fdf41944cabf0a3381c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 09:57:22 +0000 Subject: [PATCH 0824/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c246cdde3..5fadb6284 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,10 @@ Hari Sekhon - DevOps Python Tools [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) + +[![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) -[![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) From 9ab3b5493e7a3ba2af37bc51313b351c42a7157c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 11:18:57 +0000 Subject: [PATCH 0825/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index dac327ef0..13e8bf963 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit dac327ef02f570cd08b426a4e9e90a97d0171357 +Subproject commit 13e8bf96340419bd7ec9d1078bba9cf1226105f4 From 8389af956d100ecaf4b3e2a09ab0d787cd84d961 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 11:18:57 +0000 Subject: [PATCH 0826/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ade8241d3..fed3bae88 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ade8241d30c3f7049c65d6de6fb6df1dad273b83 +Subproject commit fed3bae883262763f5587a9154b4324557d5ff4a From e645990ce415172717b276e80e35239d5a057385 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 11:19:26 +0000 Subject: [PATCH 0827/2779] updated Makefile --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index f77f6f741..1b19d053d 100755 --- a/Makefile +++ b/Makefile @@ -51,6 +51,11 @@ build: @echo DevOps Python Tools Build @echo ========================= + type -P python + python -V + + pip -V + $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python From 39daf99871e1c5003469747e95adc08e6e648486 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 11:39:45 +0000 Subject: [PATCH 0828/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fed3bae88..e5c3259a1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fed3bae883262763f5587a9154b4324557d5ff4a +Subproject commit e5c3259a18e3d2439306d7b431e7a57aefe4d83d From fd8a41a0f6958cbff00de239089b2ba805f13100 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 13:15:49 +0000 Subject: [PATCH 0829/2779] updated Makefile --- Makefile | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1b19d053d..ea6b18d4a 100755 --- a/Makefile +++ b/Makefile @@ -29,6 +29,9 @@ # =================== +# would fail bootstrapping on Alpine +#SHELL := /usr/bin/env bash + ifneq ("$(wildcard bash-tools/Makefile.in)", "") include bash-tools/Makefile.in endif @@ -51,10 +54,11 @@ build: @echo DevOps Python Tools Build @echo ========================= - type -P python - python -V - - pip -V + # executing in sh where type is not available + #type -P python + which python || : + python -V || : + pip -V || : $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi @@ -63,6 +67,12 @@ build: if type apt-get 2>/dev/null; then $(MAKE) apt-packages-extra; fi $(MAKE) python + # executing in sh where type is not available + #type -P python + which python + python -V + pip -V + .PHONY: init init: git submodule update --init --recursive From 9e4de76ac9e70af9cbd2e5e3b856c61bf5e13435 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 13:16:00 +0000 Subject: [PATCH 0830/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 13e8bf963..a9a3d77bc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 13e8bf96340419bd7ec9d1078bba9cf1226105f4 +Subproject commit a9a3d77bcfbd289454cfe5bffbae104692707ca9 From 2945a1cf3a1c00b115f15e85667ba2ce4d3c7cf4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 13:16:00 +0000 Subject: [PATCH 0831/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e5c3259a1..dab1fcb73 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e5c3259a18e3d2439306d7b431e7a57aefe4d83d +Subproject commit dab1fcb739156831ab71d8e07dd55d92884423ae From 64c7517f58446cf738b7fce32329446acc7e1f95 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 14:39:37 +0000 Subject: [PATCH 0832/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a9a3d77bc..b42c6835d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a9a3d77bcfbd289454cfe5bffbae104692707ca9 +Subproject commit b42c6835df3b15c2eafbb13f6356135ebad3a426 From 84597243f3c4b748757cef575a7f97233b7d6d6e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 14:39:38 +0000 Subject: [PATCH 0833/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index dab1fcb73..5ee1fa9fb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit dab1fcb739156831ab71d8e07dd55d92884423ae +Subproject commit 5ee1fa9fb594ebe05693e1099c99fb905a5e0e1c From 9f6be2e987d64e2db3e8420790bfaa9911c467ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 15:09:53 +0000 Subject: [PATCH 0834/2779] updated Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ea6b18d4a..186c1928b 100755 --- a/Makefile +++ b/Makefile @@ -100,7 +100,8 @@ python: @bash-tools/python_pip_install.sh snakebite[kerberos] || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi + #@if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi + NO_FAIL=1 bash-tools/python_pip_install.sh "avro-python3" @# for impyla @#$(SUDO_PIP) pip install --upgrade setuptools || : From 910cca2f66e4ffe2ce130ed57c06836d29d8fb88 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 15:48:16 +0000 Subject: [PATCH 0835/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 186c1928b..066ba0940 100755 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ python: # Python >= 3.4 - try but accept failure in case we're not on the right version of Python #@if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi - NO_FAIL=1 bash-tools/python_pip_install.sh "avro-python3" + bash-tools/python_pip_install.sh "avro-python3" || : @# for impyla @#$(SUDO_PIP) pip install --upgrade setuptools || : From 957c65e9c24505890a6e37653ef66035711eb43d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:41 +0000 Subject: [PATCH 0836/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b42c6835d..fb1dd428f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b42c6835df3b15c2eafbb13f6356135ebad3a426 +Subproject commit fb1dd428fac0ae1d284793bee116f8a413970104 From 81deb21b1a0227e5e97e614028d29ea86ddd4f81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:41 +0000 Subject: [PATCH 0837/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5ee1fa9fb..e91e70074 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5ee1fa9fb594ebe05693e1099c99fb905a5e0e1c +Subproject commit e91e700743944926e8aed036a8808348b015ea1f From 755840dc7c47a8b2636f29879715654e0e14c2b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:50 +0000 Subject: [PATCH 0838/2779] renamed ci_alpine.yaml to alpine.yaml --- .github/workflows/{ci_alpine.yaml => alpine.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_alpine.yaml => alpine.yaml} (100%) diff --git a/.github/workflows/ci_alpine.yaml b/.github/workflows/alpine.yaml similarity index 100% rename from .github/workflows/ci_alpine.yaml rename to .github/workflows/alpine.yaml From 0285832e7dba620d6017431b79927274ff6bebaf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:50 +0000 Subject: [PATCH 0839/2779] renamed ci_centos.yaml to centos.yaml --- .github/workflows/{ci_centos.yaml => centos.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_centos.yaml => centos.yaml} (100%) diff --git a/.github/workflows/ci_centos.yaml b/.github/workflows/centos.yaml similarity index 100% rename from .github/workflows/ci_centos.yaml rename to .github/workflows/centos.yaml From 1b4d3dd168ebc85523a8246e6b38fbe3a9c7df7a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:51 +0000 Subject: [PATCH 0840/2779] renamed ci_mac.yaml to mac.yaml --- .github/workflows/{ci_mac.yaml => mac.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_mac.yaml => mac.yaml} (100%) diff --git a/.github/workflows/ci_mac.yaml b/.github/workflows/mac.yaml similarity index 100% rename from .github/workflows/ci_mac.yaml rename to .github/workflows/mac.yaml From edc95ea402ff0381dcba4e04f99976a57b5f87d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:51 +0000 Subject: [PATCH 0841/2779] renamed ci_python_2.7.yaml to python_2.7.yaml --- .github/workflows/{ci_python_2.7.yaml => python_2.7.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_python_2.7.yaml => python_2.7.yaml} (100%) diff --git a/.github/workflows/ci_python_2.7.yaml b/.github/workflows/python_2.7.yaml similarity index 100% rename from .github/workflows/ci_python_2.7.yaml rename to .github/workflows/python_2.7.yaml From 264271d302ecccd0ed62e6fb0d042cd640ffab94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:52 +0000 Subject: [PATCH 0842/2779] renamed ci_python_3.6.yaml to python_3.6.yaml --- .github/workflows/{ci_python_3.6.yaml => python_3.6.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_python_3.6.yaml => python_3.6.yaml} (100%) diff --git a/.github/workflows/ci_python_3.6.yaml b/.github/workflows/python_3.6.yaml similarity index 100% rename from .github/workflows/ci_python_3.6.yaml rename to .github/workflows/python_3.6.yaml From fbd0048d920e96c37d0b8038798f6f2b0963701a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:24:52 +0000 Subject: [PATCH 0843/2779] renamed ci_ubuntu.yaml to ubuntu.yaml --- .github/workflows/{ci_ubuntu.yaml => ubuntu.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_ubuntu.yaml => ubuntu.yaml} (100%) diff --git a/.github/workflows/ci_ubuntu.yaml b/.github/workflows/ubuntu.yaml similarity index 100% rename from .github/workflows/ci_ubuntu.yaml rename to .github/workflows/ubuntu.yaml From 39a29c8f334ee476b71116238eb5e4a5226b14b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:25:01 +0000 Subject: [PATCH 0844/2779] renamed python_2.7.yaml to python2.yaml --- .github/workflows/{python_2.7.yaml => python2.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python_2.7.yaml => python2.yaml} (100%) diff --git a/.github/workflows/python_2.7.yaml b/.github/workflows/python2.yaml similarity index 100% rename from .github/workflows/python_2.7.yaml rename to .github/workflows/python2.yaml From 60cb4128543949e055eb052b4d3d050a106f3677 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:25:06 +0000 Subject: [PATCH 0845/2779] renamed python_3.6.yaml to python3.yaml --- .github/workflows/{python_3.6.yaml => python3.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python_3.6.yaml => python3.yaml} (100%) diff --git a/.github/workflows/python_3.6.yaml b/.github/workflows/python3.yaml similarity index 100% rename from .github/workflows/python_3.6.yaml rename to .github/workflows/python3.yaml From a640851fbf5cb8f699b425b0f8b43f32ab5a58a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:28:12 +0000 Subject: [PATCH 0846/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 1b5a0f43c..54663fb45 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -29,7 +29,7 @@ jobs: #name: build timeout-minutes: 10 runs-on: ubuntu-latest - container: harisekhon/alpine-dev + container: alpine steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 From 564e49e7ce1aa2cc41fef3446020e1a962940d10 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:28:19 +0000 Subject: [PATCH 0847/2779] updated centos.yaml --- .github/workflows/centos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 1d8071032..ff6509ac5 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -29,7 +29,7 @@ jobs: #name: build timeout-minutes: 10 runs-on: ubuntu-latest - container: harisekhon/centos-dev + container: centos steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 From ee5c0d4822a20445e2aa3c914336135a17d6cbde Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:44:42 +0000 Subject: [PATCH 0848/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 54663fb45..967ca2840 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -38,6 +38,7 @@ jobs: ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release + apk add --no-cache git make && cd /tmp && git clone https://github.com/harisekhon/devops-python-tools && cd devops-python-tools && From 7798e46f73a758be2d25738108744245670d6b88 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:46:56 +0000 Subject: [PATCH 0849/2779] updated centos.yaml --- .github/workflows/centos.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index ff6509ac5..44b4759d1 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -38,6 +38,7 @@ jobs: ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release + yum install -y git make && cd /tmp && git clone https://github.com/harisekhon/devops-python-tools && cd devops-python-tools && From 2c8efb22b2d814baacfa6fd52c3cb452272bce07 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:51:41 +0000 Subject: [PATCH 0850/2779] added .github/workflows/centos6.yaml --- .github/workflows/centos6.yaml | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/centos6.yaml diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml new file mode 100644 index 000000000..3fc59deba --- /dev/null +++ b/.github/workflows/centos6.yaml @@ -0,0 +1,45 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI CentOS 6 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: centos:6 + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: build & test + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + yum install -y git make && + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools && + cd devops-python-tools && + make build test From 112b87f85a39d0b2ef5b43609404191e178851cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 16:52:01 +0000 Subject: [PATCH 0851/2779] added .github/workflows/centos7.yaml --- .github/workflows/centos7.yaml | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/centos7.yaml diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml new file mode 100644 index 000000000..853d33df3 --- /dev/null +++ b/.github/workflows/centos7.yaml @@ -0,0 +1,45 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI CentOS 7 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: centos:7 + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: build & test + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + yum install -y git make && + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools && + cd devops-python-tools && + make build test From 661a8e5453e7d8e2a363f3c3e9ed6cc966e534ef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2020 20:51:17 +0000 Subject: [PATCH 0852/2779] updated rpm-packages.txt --- setup/rpm-packages.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index 281cfb022..cf3109c39 100644 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -22,7 +22,7 @@ curl iputils # for htpasswd for docker registry authenticated checks -https-tools +httpd-tools # needed to build pyhs2 # libgsasl-devel saslwrapper-devel From fde1f6695e752ef3cced32c5f461b7defcfdcc9f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:29:10 +0000 Subject: [PATCH 0853/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 967ca2840..9b188de5a 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -33,13 +33,21 @@ jobs: steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - - name: build & test + - name: install git & make run: | ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release - apk add --no-cache git make && + apk add --no-cache git make + - name: git clone + run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools && - cd devops-python-tools && - make build test + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From f3d23a006bbe9f1aa61992f31033af6cadcfb442 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:29:27 +0000 Subject: [PATCH 0854/2779] updated centos.yaml --- .github/workflows/centos.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 44b4759d1..daf5d8d57 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -33,13 +33,21 @@ jobs: steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - - name: build & test + - name: install git & make run: | ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release - yum install -y git make && + yum install -y git make + - name: git clone + run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools && - cd devops-python-tools && - make build test + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From 65f44e2caa1628e61c982a3f9bc1fec7b06dfde8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:29:37 +0000 Subject: [PATCH 0855/2779] updated centos6.yaml --- .github/workflows/centos6.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml index 3fc59deba..5856a69d4 100644 --- a/.github/workflows/centos6.yaml +++ b/.github/workflows/centos6.yaml @@ -33,13 +33,21 @@ jobs: steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - - name: build & test + - name: install git & make run: | ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release - yum install -y git make && + yum install -y git make + - name: git clone + run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools && - cd devops-python-tools && - make build test + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From cacc9e4440dc67384a3d518a105ff61587ff596a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:29:49 +0000 Subject: [PATCH 0856/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 853d33df3..f92b232d4 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -33,13 +33,21 @@ jobs: steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - - name: build & test + - name: install git & make run: | ls -l /.dockerenv echo "pwd is $PWD" cat /etc/*release - yum install -y git make && + yum install -y git make + - name: git clone + run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools && - cd devops-python-tools && - make build test + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From cc369e4176c1ab2007c3575e6bcf0d7d3e330ac1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:30:11 +0000 Subject: [PATCH 0857/2779] added centos8.yaml --- .github/workflows/centos8.yaml | 53 ++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/centos8.yaml diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml new file mode 100644 index 000000000..2016b60d6 --- /dev/null +++ b/.github/workflows/centos8.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI CentOS 8 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: centos:8 + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + yum install -y git make + - name: git clone + run: | + cd /tmp && + git clone https://github.com/harisekhon/devops-python-tools + - name: build + run: | + cd /tmp/devops-python-tools && + make + - name: test + run: | + cd /tmp/devops-python-tools && + make test From aafdeb56f4dcfab409a897028ac322db9ab6f1b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:30:24 +0000 Subject: [PATCH 0858/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index a919dbf9d..5b419ce79 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -40,5 +40,7 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - - name: build & test - run: make build test + - name: build + run: make + - name: test + run: make test From c32147809deacb5aeac978225aaee387dd57030f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Feb 2020 17:30:34 +0000 Subject: [PATCH 0859/2779] updated mac.yaml --- .github/workflows/mac.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 36f71e43d..db5e0d0da 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -37,5 +37,7 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - - name: build & test - run: make build test + - name: build + run: make + - name: test + run: make test From e89bed8072e12f6651dc0b0916b9dd9230376c56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 8 Feb 2020 09:20:11 +0000 Subject: [PATCH 0860/2779] updated README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 5fadb6284..9185ade80 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,9 @@ Hari Sekhon - DevOps Python Tools [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) +[![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) +[![CI CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+7%22) +[![CI CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+8%22) [![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) [![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) [![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) From aa3767659dc2b40c52ec7eff16a034ec5bbc264d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 09:52:54 +0000 Subject: [PATCH 0861/2779] updated rpm-packages-optional.txt --- setup/rpm-packages-optional.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt index f94b6019d..11e757a6f 100644 --- a/setup/rpm-packages-optional.txt +++ b/setup/rpm-packages-optional.txt @@ -14,3 +14,8 @@ # ============================================================================ # yamllint + +# CentOS <= 7 +snappy-devel +# CentOS 8 +csnappy-devel From 1a6b3aaf49c182fcd2f8ec22dbbc061ed08360dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 09:53:17 +0000 Subject: [PATCH 0862/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index a0015d809..c812c26ed 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -18,5 +18,7 @@ gcc-c++ cyrus-sasl-devel krb5-devel openssl-devel + +# moved to optional to account for changed package names on CentOS 8 # needed to build python-snappy for avro module -snappy-devel +#snappy-devel From a6140320ddc6be4f2dfcdfc4860675797ad095b2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 09:57:44 +0000 Subject: [PATCH 0863/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fb1dd428f..8bfc4536f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fb1dd428fac0ae1d284793bee116f8a413970104 +Subproject commit 8bfc4536f6bf60c2e73793f266a7e509b60a3b05 From fa3b4f9d8f44da2dbabd0455da0ee2fd0324218b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 09:57:44 +0000 Subject: [PATCH 0864/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e91e70074..de5d01494 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e91e700743944926e8aed036a8808348b015ea1f +Subproject commit de5d01494f4ba302020254820aed850ff71e0778 From 288d17808971029be0c93e2532a8dc9bc453700b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 10:13:38 +0000 Subject: [PATCH 0865/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index c812c26ed..6dd8fa477 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -17,6 +17,7 @@ gcc-c++ # needed to build python-krbV and cloudera/thrift_sasl cyrus-sasl-devel krb5-devel +openldap-devel openssl-devel # moved to optional to account for changed package names on CentOS 8 From 0ba717d8b2325ecc524c80096e8883a61ce5b903 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 23:44:24 +0000 Subject: [PATCH 0866/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8bfc4536f..1b02ade81 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8bfc4536f6bf60c2e73793f266a7e509b60a3b05 +Subproject commit 1b02ade8112a825103a88487ddc4184c2f951247 From 8ccf3fb407916312d2a540acae323c541e1dbf97 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 23:44:24 +0000 Subject: [PATCH 0867/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index de5d01494..93798d25a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit de5d01494f4ba302020254820aed850ff71e0778 +Subproject commit 93798d25afc89abff39dc42c411356c7884e2f19 From c8403d408fbb21165cb8c727adfa32a62f0f7ee0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Feb 2020 23:44:39 +0000 Subject: [PATCH 0868/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b02ade81..05de967ac 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b02ade8112a825103a88487ddc4184c2f951247 +Subproject commit 05de967ac225d60d9973c827c72674fd8aca7f20 From 42bdb181b22090145d17dfdc7ff11eb273fd5619 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 10:40:44 +0000 Subject: [PATCH 0869/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 93798d25a..271e16ac5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 93798d25afc89abff39dc42c411356c7884e2f19 +Subproject commit 271e16ac5d0b2d0916bd11cf3541e55db23c1f3e From 963a89e466cd79c3cf70be6f4a9c3a2d0b018ea0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 10:51:18 +0000 Subject: [PATCH 0870/2779] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 066ba0940..ef9f2ce1b 100755 --- a/Makefile +++ b/Makefile @@ -54,8 +54,8 @@ build: @echo DevOps Python Tools Build @echo ========================= - # executing in sh where type is not available - #type -P python + @# executing in sh where type is not available + @#type -P python which python || : python -V || : pip -V || : From 21b468c8fe7715b09c7256b75c65d709e4c7a472 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:13:33 +0000 Subject: [PATCH 0871/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 271e16ac5..379794ebf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 271e16ac5d0b2d0916bd11cf3541e55db23c1f3e +Subproject commit 379794ebf7510100cf07a3d7af5b4127181738d1 From 47e6d69567d7740d58493c043d1d09ec6c97bd80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:25:31 +0000 Subject: [PATCH 0872/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 05de967ac..91a889ad7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 05de967ac225d60d9973c827c72674fd8aca7f20 +Subproject commit 91a889ad7df753e49d1e8556ede183a11b77646c From 72ede886b74840c92afb8107361d688b083fb80f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:25:32 +0000 Subject: [PATCH 0873/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 379794ebf..5fd874ae1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 379794ebf7510100cf07a3d7af5b4127181738d1 +Subproject commit 5fd874ae1a2891fdfdcbc7c22d2608ce19b39a6d From 3bb834b12a0510862cede283ffc14c96ee16bf13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:25:47 +0000 Subject: [PATCH 0874/2779] updated test_welcome.sh --- tests/test_welcome.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_welcome.sh b/tests/test_welcome.sh index 1be838108..9deb7ad8a 100755 --- a/tests/test_welcome.sh +++ b/tests/test_welcome.sh @@ -20,6 +20,11 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; # shellcheck disable=SC1091 -#. ./tests/utils.sh +. ./tests/utils.sh + +# Fedora doesn't have /var/log/wtmp +if ! [ -f /var/log/wtmp ]; then + $sudo touch /var/log/wtmp || : +fi ./welcome.py "$@" From f7ce2417a1557277c87daaac0fd8b0d8aaad708a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:43:29 +0000 Subject: [PATCH 0875/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 91a889ad7..d117e0672 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 91a889ad7df753e49d1e8556ede183a11b77646c +Subproject commit d117e067235a8078cd93b0b4d9425482e2c78150 From 53e4e10b3c5cd659b58a09429b23f77669d22451 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 11:43:30 +0000 Subject: [PATCH 0876/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5fd874ae1..6eef2ab50 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5fd874ae1a2891fdfdcbc7c22d2608ce19b39a6d +Subproject commit 6eef2ab5001d5ff66cecb3cbd70957b5b841e826 From 4b649cbfe4531ecc26541d2f2270290a21047ddd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 14:09:54 +0000 Subject: [PATCH 0877/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d117e0672..a83041be4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d117e067235a8078cd93b0b4d9425482e2c78150 +Subproject commit a83041be449870e9c98ade290ed837fdbae3e6e5 From 4d2fd5e9405fb29a1cb955156a9aea713e366c13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 14:09:54 +0000 Subject: [PATCH 0878/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6eef2ab50..9bf26312e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6eef2ab5001d5ff66cecb3cbd70957b5b841e826 +Subproject commit 9bf26312e298b90bab29eadfb5d2238778dbbe4a From 9f8ad47242b654a952b66aaa5e047858f9be6667 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:13:38 +0000 Subject: [PATCH 0879/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a83041be4..c8bfc3bab 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a83041be449870e9c98ade290ed837fdbae3e6e5 +Subproject commit c8bfc3bab2d1625eb10d5fa8a37c5e908bc6d8bc From ebee286fd42e8da3dce1c31c389ae5a5d5a35aa7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:13:39 +0000 Subject: [PATCH 0880/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9bf26312e..6d460ffcb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9bf26312e298b90bab29eadfb5d2238778dbbe4a +Subproject commit 6d460ffcb0d520ccf1b07d0e0a4762fff45da61d From f701b75d41c1d1e73367e83d68e83132d4c17b93 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:15:25 +0000 Subject: [PATCH 0881/2779] updated rpm-packages.txt --- setup/rpm-packages.txt | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index cf3109c39..cf5f0222b 100644 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -15,12 +15,6 @@ java -# needed for tests/test_spark* and ambari_cancel_all_requests.sh -curl - -# for ping mode of find_active_server.py -iputils - # for htpasswd for docker registry authenticated checks httpd-tools @@ -28,8 +22,17 @@ httpd-tools # libgsasl-devel saslwrapper-devel #cyrus-sasl-devel +# ===================================== +# installed by bash-tools submodule now + +# needed for tests/test_spark* and ambari_cancel_all_requests.sh +#curl + +# for ping mode of find_active_server.py +#iputils + # for anonymize_parallel.sh -parallel +#parallel -unzip -zip +#unzip +#zip From e886aa4a9f9d8ddb2b8f7e77c02c5d8de70a950a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:17:27 +0000 Subject: [PATCH 0882/2779] updated deb-packages.txt --- setup/deb-packages.txt | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index c605fdd27..d7dc245df 100644 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -13,22 +13,9 @@ # Deb Package Requirements # ============================================================================ # -# needed for tests/test_spark* and ambari_cancel_all_requests.sh -curl - -# either of these should do to ensure ping command is present for find_active_server.py's ping mode -iputils-ping -#inetutils-ping - # for htpasswd for docker registry authenticated checks apache2-utils -# for anonymize_parallel.sh -parallel - -# needed for serf test's 'uptime' command -procps - # installs 343MB of dependencies - install by hand if needed #ffmpeg @@ -41,5 +28,21 @@ procps #which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk -zip -unzip +# ===================================== +# installed by bash-tools submodule now + +# needed for tests/test_spark* and ambari_cancel_all_requests.sh +#curl + +# either of these should do to ensure ping command is present for find_active_server.py's ping mode +#iputils-ping +##inetutils-ping + +# for anonymize_parallel.sh +#parallel + +# needed for serf test's 'uptime' command +#procps + +#zip +#unzip From 94bc98974794dc15fbf5b97a814f11e4b0b217b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:18:55 +0000 Subject: [PATCH 0883/2779] updated apk-packages.txt --- setup/apk-packages.txt | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt index fc6956249..bccca7733 100644 --- a/setup/apk-packages.txt +++ b/setup/apk-packages.txt @@ -13,24 +13,27 @@ # Alpine Package Requirements # ============================================================================ # -# full mktemp needed for json tests -# full split needed for anonymize_parallel.sh -coreutils - -grep - -# needed for tests/test_spark* and ambari_cancel_all_requests.sh -curl - # for htpasswd for docker registry authenticated checks apache2-utils -# for anonymize_parallel.sh -parallel - #which java || $(SUDO) apk add openjdk8-jre-base # Spark Java Py4J gets java linking error without this #if [ -f /lib/libc.musl-x86_64.so.1 ]; then [ -e /lib/ld-linux-x86-64.so.2 ] || ln -sv /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2; fi -zip +# ===================================== +# installed by bash-tools submodule now + +# full mktemp needed for json tests +# full split needed for anonymize_parallel.sh +#coreutils + +# needed for tests/test_spark* and ambari_cancel_all_requests.sh +#curl + +#grep + +# for anonymize_parallel.sh +#parallel + +#zip From c668b3cb6cd905f60e9c35700a5ad22ee8fa0ed5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:19:29 +0000 Subject: [PATCH 0884/2779] updated apk-packages-dev.txt --- setup/apk-packages-dev.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt index f8738775b..67e6f895d 100644 --- a/setup/apk-packages-dev.txt +++ b/setup/apk-packages-dev.txt @@ -15,4 +15,6 @@ openldap-dev snappy-dev -unzip + +# installed by bash-tools submodule now +#unzip From d667dbd4661bbbe2f8aa367914feb21db7adfca8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:19:56 +0000 Subject: [PATCH 0885/2779] updated brew-packages.txt --- setup/brew-packages.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index fa25317ba..03a0a78a5 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -14,8 +14,9 @@ # Mac OS X - Homebrew Package Requirements # ============================================================================ # +# installed by bash-tools submodule now # for anonymize_parallel.sh -parallel +#parallel parquet-tools snappy From fb2903132ca2ff83d4e62cadc39012316c41c899 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:20:21 +0000 Subject: [PATCH 0886/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c8bfc3bab..fab97843d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c8bfc3bab2d1625eb10d5fa8a37c5e908bc6d8bc +Subproject commit fab97843df459a58170c70b2d848223c9a8af8a5 From ec3720c2d78c9797f602008a25de3294876e1b2e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:24:56 +0000 Subject: [PATCH 0887/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6d460ffcb..7db4ac021 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6d460ffcb0d520ccf1b07d0e0a4762fff45da61d +Subproject commit 7db4ac0217fa34b38884d768fd3de38b9463117d From 9b30771321beb067c1705847f2207e2aab005741 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:30:51 +0000 Subject: [PATCH 0888/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fab97843d..2b0439d88 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fab97843df459a58170c70b2d848223c9a8af8a5 +Subproject commit 2b0439d886154c961e51154a3e73ae4717493b13 From aa81f784bd9e8cc7502e634e898a2f551c00a5b9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 11 Feb 2020 17:30:51 +0000 Subject: [PATCH 0889/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7db4ac021..1f0b5a030 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7db4ac0217fa34b38884d768fd3de38b9463117d +Subproject commit 1f0b5a030216b9e08e63016ffeeb368256badb79 From 8c505b12f44abb2a73787055decaa361f87ae749 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 12:08:37 +0000 Subject: [PATCH 0890/2779] updated impala_tables_locations.py --- impala_tables_locations.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/impala_tables_locations.py b/impala_tables_locations.py index 1182164c6..63bae3e0f 100755 --- a/impala_tables_locations.py +++ b/impala_tables_locations.py @@ -19,6 +19,13 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes +Caveats: + + Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't + + Impala is faster than Hive for the first hundred or so tables but then slows down + so if you have a lot of tables I recommend you use the Hive version of this instead + Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 2820ce8d9e331b6b55a15aac4a01930131e85df7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 12:08:52 +0000 Subject: [PATCH 0891/2779] updated impala_tables_metadata.py --- impala_tables_metadata.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 8182f74e1..792be37d4 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -19,6 +19,13 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes +Caveats: + + Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't + + Impala is faster than Hive for the first hundred or so tables but then slows down + so if you have a lot of tables I recommend you use the Hive version of this instead + Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 1089f2645a6e1db24ce20f858ece60c09ef0adc9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 13:53:15 +0000 Subject: [PATCH 0892/2779] updated impala_tables_locations.py --- impala_tables_locations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_locations.py b/impala_tables_locations.py index 63bae3e0f..12c0a93f2 100755 --- a/impala_tables_locations.py +++ b/impala_tables_locations.py @@ -23,7 +23,7 @@ Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't - Impala is faster than Hive for the first hundred or so tables but then slows down + Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From d470418d29d4e66253dcbf8aa9e7f62a71b1c58c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 13:53:20 +0000 Subject: [PATCH 0893/2779] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 792be37d4..2baa580e8 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -23,7 +23,7 @@ Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't - Impala is faster than Hive for the first hundred or so tables but then slows down + Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From aba0a3aa0a01c9680dfbb142da16e2a7268269df Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 14:03:40 +0000 Subject: [PATCH 0894/2779] updated hive_tables_metadata.py --- hive_tables_metadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index ccf66fc2a..04b89e728 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class HiveTablesMetadata(HiveForEachTable): @@ -94,7 +94,7 @@ def process_options(self): # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): log.info("describing table '%s.%s'", database, table) - location = 'UNKNOWN' + field = 'UNKNOWN' with conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) @@ -103,9 +103,9 @@ def execute(self, conn, database, table, query): table_cursor.execute(query.format(table=table)) for row in table_cursor: if self.field.search(row[0]): - location = row[1] + field = row[1] break - print('{db}.{table}\t{location}'.format(db=database, table=table, location=location)) + print('{db}.{table}\t{field}'.format(db=database, table=table, field=field)) if __name__ == '__main__': From df8aee7355e3082df710c4e4aafec6fcaea99f0e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 23:20:56 +0000 Subject: [PATCH 0895/2779] updated README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9185ade80..c701d780c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) +[![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) +[![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) +[![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) [![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) From 23632588d56824170c41f7853357b5b11ff41ddf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 23:21:30 +0000 Subject: [PATCH 0896/2779] added ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/ubuntu_14.04.yaml diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml new file mode 100644 index 000000000..c03f7072e --- /dev/null +++ b/.github/workflows/ubuntu_14.04.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu 14.04 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: ubuntu:14.04 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 56be868bee97a34d0f4454f3229fa8e197f764b2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 23:21:30 +0000 Subject: [PATCH 0897/2779] added ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/ubuntu_16.04.yaml diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml new file mode 100644 index 000000000..714dbbaf7 --- /dev/null +++ b/.github/workflows/ubuntu_16.04.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu 16.04 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: ubuntu:16.04 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From fcffe68c0b1b4732925e2cadd9d395c81790ca96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 Feb 2020 23:21:30 +0000 Subject: [PATCH 0898/2779] added ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/ubuntu_18.04.yaml diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml new file mode 100644 index 000000000..d3ab2752c --- /dev/null +++ b/.github/workflows/ubuntu_18.04.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu 18.04 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: ubuntu:18.04 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From b11e7dbd5049a56fccfa5ff5eef786a6675d1324 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 14:31:58 +0000 Subject: [PATCH 0899/2779] added auto-determining repo from local git remotes --- travis_last_log.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/travis_last_log.py b/travis_last_log.py index b9b6843f2..5005c3417 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -61,6 +61,7 @@ #import string import sys import traceback +import git srcdir = os.path.abspath(os.path.dirname(__file__)) libdir = os.path.join(srcdir, 'pylib') sys.path.append(libdir) @@ -76,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.6.0' class TravisLastBuildLog(CLI): @@ -147,7 +148,9 @@ def process_options(self): self.repo = travis_user + self.repo validate_chars(self.repo, 'repo', r'\/\w\.-') else: - self.usage('--job-id / --repo not specified') + self.repo = self.get_local_repo_name() + if not self.repo: + self.usage('--job-id / --repo not specified') validate_alnum(self.travis_token, 'travis token') self.headers['Authorization'] = 'token {0}'.format(self.travis_token) self.num = self.get_opt('num') @@ -163,6 +166,18 @@ def process_options(self): #if not self.color and not (sys.__stdin__.isatty() and sys.__stdout__.isatty()): # self.plaintext = True + @staticmethod + def get_local_repo_name(): + try: + _ = git.Repo('.') + for remote in _.remotes: + for url in remote.urls: + repo = '/'.join(url.split('/')[-2:]) + log.debug('determined repo to be {} from remotes'.format(repo)) + return repo + except git.InvalidGitRepositoryError: + log.debug('failed to determine git repository locally: %s', _) + def run(self): if self.job_id: self.print_log(job_id=self.job_id) From 0a2d66ca6df74608a7a57e6adb7144412a24800d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 14:37:47 +0000 Subject: [PATCH 0900/2779] added auto-determining repo from local git remotes --- travis_debug_session.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/travis_debug_session.py b/travis_debug_session.py index 6812eeead..eb2557d7d 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -48,6 +48,7 @@ import sys import time import traceback +import git try: import requests except ImportError: @@ -68,7 +69,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.4' +__version__ = '0.9.0' class TravisDebugSession(CLI): @@ -151,10 +152,24 @@ def process_options(self): self.repo = travis_user + self.repo validate_chars(self.repo, 'repo', r'\/\w\.-') else: - self.usage('--job-id / --repo not specified') + self.repo = self.get_local_repo_name() + if not self.repo: + self.usage('--job-id / --repo not specified') validate_alnum(self.travis_token, 'travis token') self.headers['Authorization'] = 'token {0}'.format(self.travis_token) + @staticmethod + def get_local_repo_name(): + try: + _ = git.Repo('.') + for remote in _.remotes: + for url in remote.urls: + repo = '/'.join(url.split('/')[-2:]) + log.debug('determined repo to be {} from remotes'.format(repo)) + return repo + except git.InvalidGitRepositoryError: + log.debug('failed to determine git repository locally: %s', _) + def run(self): if not self.job_id: if self.repo: From a659c16927d2520747e23e41035cc52b9a3f756c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 15:12:00 +0000 Subject: [PATCH 0901/2779] added .github/workflows/fedora.yaml --- .github/workflows/fedora.yaml | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/fedora.yaml diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml new file mode 100644 index 000000000..8ae839658 --- /dev/null +++ b/.github/workflows/fedora.yaml @@ -0,0 +1,55 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Fedora + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: fedora + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + yum install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 18f9be55279662e8c25e2ecaa6d005df5fac78f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 15:19:24 +0000 Subject: [PATCH 0902/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c701d780c..cf203535d 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Hari Sekhon - DevOps Python Tools [![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) [![CI CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+7%22) [![CI CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+8%22) +[![CI Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Fedora%22) [![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) [![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) [![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) From b867dd3b94a55ca32c170c867d68e0cafa37e420 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:48 +0000 Subject: [PATCH 0903/2779] sync'd workflows from bash-tools --- .github/workflows/alpine.yaml | 8 +++++--- .github/workflows/centos.yaml | 8 +++++--- .github/workflows/centos6.yaml | 10 +++++++--- .github/workflows/centos7.yaml | 8 +++++--- .github/workflows/centos8.yaml | 8 +++++--- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 9b188de5a..7988da349 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -30,6 +30,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: alpine + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +44,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index daf5d8d57..8d3c00564 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -30,6 +30,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: centos + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +44,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml index 5856a69d4..f2e27b9d4 100644 --- a/.github/workflows/centos6.yaml +++ b/.github/workflows/centos6.yaml @@ -11,6 +11,8 @@ # https://www.linkedin.com/in/harisekhon # +# Not supporting RHEL6 any more because it doesn't have GNU parallel package + name: CI CentOS 6 #env: @@ -30,6 +32,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: centos:6 + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +46,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index f92b232d4..3939b04f2 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -30,6 +30,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: centos:7 + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +44,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 2016b60d6..407ead97f 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -30,6 +30,8 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest container: centos:8 + env: + repo: devops-python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 @@ -42,12 +44,12 @@ jobs: - name: git clone run: | cd /tmp && - git clone https://github.com/harisekhon/devops-python-tools + git clone "https://github.com/harisekhon/$repo" - name: build run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make - name: test run: | - cd /tmp/devops-python-tools && + cd "/tmp/$repo" && make test From 6d1fd6655d5fe66e60264698c60db17a91dc79b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:51 +0000 Subject: [PATCH 0904/2779] added alpine_3.yaml --- .github/workflows/alpine_3.yaml | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/alpine_3.yaml diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml new file mode 100644 index 000000000..da8a373a3 --- /dev/null +++ b/.github/workflows/alpine_3.yaml @@ -0,0 +1,55 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Alpine 3 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: alpine:3 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apk add --no-cache git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From a3e2da25d098c40c31e22f40e2004c70ab1afe15 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:51 +0000 Subject: [PATCH 0905/2779] added debian.yaml --- .github/workflows/debian.yaml | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian.yaml diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml new file mode 100644 index 000000000..43392f0c5 --- /dev/null +++ b/.github/workflows/debian.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 37dfcaefeefe51f53c539bcfefa133efaa621dec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:52 +0000 Subject: [PATCH 0906/2779] added debian_10.yaml --- .github/workflows/debian_10.yaml | 56 ++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_10.yaml diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml new file mode 100644 index 000000000..31775a833 --- /dev/null +++ b/.github/workflows/debian_10.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 10 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:10-slim + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 2a458bfb8f129de6c50103e519ff39e435986265 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:52 +0000 Subject: [PATCH 0907/2779] added debian_6.yaml --- .github/workflows/debian_6.yaml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_6.yaml diff --git a/.github/workflows/debian_6.yaml b/.github/workflows/debian_6.yaml new file mode 100644 index 000000000..f0ec8860c --- /dev/null +++ b/.github/workflows/debian_6.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 6 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:6 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + echo VERSION: ; cat /etc/*release /etc/*version 2>/dev/null || : + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 8a4c6438a9264cf4a5951ea76f9b38370580a995 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:53 +0000 Subject: [PATCH 0908/2779] added debian_7.yaml --- .github/workflows/debian_7.yaml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_7.yaml diff --git a/.github/workflows/debian_7.yaml b/.github/workflows/debian_7.yaml new file mode 100644 index 000000000..640d28a79 --- /dev/null +++ b/.github/workflows/debian_7.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 7 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:7-slim + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 674f9be36ced36c905906c5ef11dbb37423dd290 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:53 +0000 Subject: [PATCH 0909/2779] added debian_8.yaml --- .github/workflows/debian_8.yaml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_8.yaml diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml new file mode 100644 index 000000000..b191b7355 --- /dev/null +++ b/.github/workflows/debian_8.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 8 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:8-slim + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From 6fab28d1465bfbd549b565d706d424c81e51641c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:15:54 +0000 Subject: [PATCH 0910/2779] added debian_9.yaml --- .github/workflows/debian_9.yaml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/debian_9.yaml diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml new file mode 100644 index 000000000..3308dec1b --- /dev/null +++ b/.github/workflows/debian_9.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Debian 9 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 10 + runs-on: ubuntu-latest + container: debian:9-slim + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: | + ls -l /.dockerenv + echo "pwd is $PWD" + cat /etc/*release + apt-get update && + apt-get install -y git make + - name: git clone + run: | + cd /tmp && + git clone "https://github.com/harisekhon/$repo" + - name: build + run: | + cd "/tmp/$repo" && + make + - name: test + run: | + cd "/tmp/$repo" && + make test From d64ea52a1c810cf006869a8577fd062bca2ffb97 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Feb 2020 17:21:05 +0000 Subject: [PATCH 0911/2779] updated README.md --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf203535d..5e6ed34a9 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,23 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) +[![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) [![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) -[![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) +[![CI Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian%22) +[![CI Debian 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+6%22) +[![CI Debian 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+7%22) +[![CI Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+8%22) +[![CI Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+9%22) +[![CI Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+10%22) [![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) [![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) [![CI CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+7%22) [![CI CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+8%22) [![CI Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Fedora%22) [![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) +[![CI Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine+3%22) [![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) [![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) From 1a7f1af1ff18abd92b62ceb1971d130393334495 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 14:11:58 +0000 Subject: [PATCH 0912/2779] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 2baa580e8..54d79dbe9 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -25,6 +25,8 @@ Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead + eg. by ~1900 tables the Hive version will overtake the Impala version and + for thousands of tables Impala actuallys runs ~1.5x slower than the Hive version overall Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From 262835ba24253c3341aee0a2edef1bc033684e45 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:00 +0000 Subject: [PATCH 0913/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2b0439d88..cee0464e9 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2b0439d886154c961e51154a3e73ae4717493b13 +Subproject commit cee0464e95f806f77b70207de51972d3f6780052 From a96c839704b4374c30500cb836d295b15a487fab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:00 +0000 Subject: [PATCH 0914/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1f0b5a030..e4434cfc6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1f0b5a030216b9e08e63016ffeeb368256badb79 +Subproject commit e4434cfc6be2e5a9b17252d0a2c0d0f98b5f3574 From ed161ee726a2a65654be01a485a677b07be621d7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:08 +0000 Subject: [PATCH 0915/2779] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5e6ed34a9..54e939f7a 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,6 @@ Hari Sekhon - DevOps Python Tools [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) [![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) [![CI Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian%22) -[![CI Debian 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+6%22) -[![CI Debian 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+7%22) [![CI Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+8%22) [![CI Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+9%22) [![CI Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+10%22) From 58783f76db75b085c2170b99eb96c98255fb480c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:12 +0000 Subject: [PATCH 0916/2779] renamed debian_6.yaml to debian_6.yamldisabled --- .github/workflows/{debian_6.yaml => debian_6.yamldisabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_6.yaml => debian_6.yamldisabled} (100%) diff --git a/.github/workflows/debian_6.yaml b/.github/workflows/debian_6.yamldisabled similarity index 100% rename from .github/workflows/debian_6.yaml rename to .github/workflows/debian_6.yamldisabled From b82102310bba71d9e4ff06398577850cdb2c572b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:30:12 +0000 Subject: [PATCH 0917/2779] renamed debian_7.yaml to debian_7.yamldisabled --- .github/workflows/{debian_7.yaml => debian_7.yamldisabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_7.yaml => debian_7.yamldisabled} (100%) diff --git a/.github/workflows/debian_7.yaml b/.github/workflows/debian_7.yamldisabled similarity index 100% rename from .github/workflows/debian_7.yaml rename to .github/workflows/debian_7.yamldisabled From a73ffa05c22013884029ec5312f207bad1c1b7d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:33:17 +0000 Subject: [PATCH 0918/2779] renamed debian_6.yamldisabled to debian_6.yaml.disabled --- .../workflows/{debian_6.yamldisabled => debian_6.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_6.yamldisabled => debian_6.yaml.disabled} (100%) diff --git a/.github/workflows/debian_6.yamldisabled b/.github/workflows/debian_6.yaml.disabled similarity index 100% rename from .github/workflows/debian_6.yamldisabled rename to .github/workflows/debian_6.yaml.disabled From c01cafbdc50b195ce378283176e8a2788315164f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:33:17 +0000 Subject: [PATCH 0919/2779] renamed debian_7.yamldisabled to debian_7.yaml.disabled --- .../workflows/{debian_7.yamldisabled => debian_7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_7.yamldisabled => debian_7.yaml.disabled} (100%) diff --git a/.github/workflows/debian_7.yamldisabled b/.github/workflows/debian_7.yaml.disabled similarity index 100% rename from .github/workflows/debian_7.yamldisabled rename to .github/workflows/debian_7.yaml.disabled From 00c9c1ebd160fd17bf2a37f33044dcc8805cfd7f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2020 23:34:37 +0000 Subject: [PATCH 0920/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e4434cfc6..ff66e8a3b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e4434cfc6be2e5a9b17252d0a2c0d0f98b5f3574 +Subproject commit ff66e8a3b243e798b507d1717c69927129e9daa1 From ff623990c621f3538442644a2bce73d2324d6c4f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 15:07:24 +0000 Subject: [PATCH 0921/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cee0464e9..46e4a2ace 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cee0464e95f806f77b70207de51972d3f6780052 +Subproject commit 46e4a2acef5c34fc17cdab9350192cb8ee5f6e18 From ba7fb678acc165a4c1d6cbf73af13c826a95a91f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 15:07:24 +0000 Subject: [PATCH 0922/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ff66e8a3b..b5a9c16a2 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ff66e8a3b243e798b507d1717c69927129e9daa1 +Subproject commit b5a9c16a2e05e5d6a645a7ccfd68de7ba1780955 From 4ae8df003113c3e6c27c91d888ece149a54bd5be Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 16:08:18 +0000 Subject: [PATCH 0923/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ef9f2ce1b..887df140f 100755 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ apk-packages-extra: .PHONY: apt-packages-extra apt-packages-extra: - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk; fi + if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y default-jdk; fi # for validate_multimedia.py # available in Alpine 2.6, 2.7 and 3.x From ddb2c860c63d8c570a0238fe29efe1ef7a7b1df6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 17:49:27 +0000 Subject: [PATCH 0924/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 5b419ce79..824b319a4 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -30,6 +30,7 @@ jobs: timeout-minutes: 10 runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, ubuntu-16.04] steps: From 905fe194874a67843dd31d7e7e00b4a06c8b690b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 23:38:28 +0000 Subject: [PATCH 0925/2779] updated mac.yaml --- .github/workflows/mac.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index db5e0d0da..7d67cd384 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -37,6 +37,8 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- + - name: brew update + run: which brew && brew update || : - name: build run: make - name: test From 66daa1e1949991b703c919216c8adb5f844cceaa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 23:40:23 +0000 Subject: [PATCH 0926/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 824b319a4..fae40bf65 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -28,11 +28,7 @@ jobs: build: #name: build timeout-minutes: 10 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, ubuntu-16.04] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/cache@v1 From 5fe291961b89527e3ed312afc6d47ab73508ab52 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Feb 2020 23:51:12 +0000 Subject: [PATCH 0927/2779] updated mac.yaml --- .github/workflows/mac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 7d67cd384..70cf40da8 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: macos-latest steps: - uses: actions/checkout@v2 From b898f46e00c3a5962642e52926de9bca8e9b6a62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 10:49:43 +0000 Subject: [PATCH 0928/2779] updated workflows --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos6.yaml | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 4 ++-- .github/workflows/debian_8.yaml | 4 ++-- .github/workflows/debian_9.yaml | 4 ++-- .github/workflows/fedora.yaml | 2 +- .github/workflows/mac.yaml | 6 +++--- .github/workflows/ubuntu.yaml | 10 +++++----- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- 16 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 7988da349..eee5180b8 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: alpine env: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index da8a373a3..45ba0ee60 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: alpine:3 env: diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 8d3c00564..b4bedd79d 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: centos env: diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml index f2e27b9d4..eb053b667 100644 --- a/.github/workflows/centos6.yaml +++ b/.github/workflows/centos6.yaml @@ -29,7 +29,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: centos:6 env: diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 3939b04f2..2b3567790 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: centos:7 env: diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 407ead97f..807ea4d85 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: centos:8 env: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 43392f0c5..c0d32c539 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -27,7 +27,7 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: debian env: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 31775a833..24499f3ec 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -27,9 +27,9 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest - container: debian:10-slim + container: debian:10 # -slim gets java install package conflicts env: repo: devops-python-tools steps: diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index b191b7355..e2cad44dd 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -27,9 +27,9 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest - container: debian:8-slim + container: debian:8 # -slim gets java install package conflicts env: repo: devops-python-tools steps: diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 3308dec1b..f2d84ffb8 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -27,9 +27,9 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest - container: debian:9-slim + container: debian:9 # -slim gets java install package conflicts env: repo: devops-python-tools steps: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 8ae839658..735a91d72 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -27,7 +27,7 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: fedora env: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 70cf40da8..3632e946c 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -1,6 +1,6 @@ # # Author: Hari Sekhon -# Date: 2020-02-04 21:39:47 +0000 (Tue, 04 Feb 2020) +# Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # @@ -34,9 +34,9 @@ jobs: - uses: actions/cache@v1 with: path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} restore-keys: | - ${{ runner.os }}-pip- + ${{ runner.os }}-pip-devops-python-tools - name: brew update run: which brew && brew update || : - name: build diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index fae40bf65..840e5553a 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -1,6 +1,6 @@ # # Author: Hari Sekhon -# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# Date: Tue Feb 4 09:53:28 2020 +0000 # # vim:ts=2:sts=2:sw=2:et # @@ -16,7 +16,7 @@ name: CI Ubuntu #env: # DEBUG: 1 -on: +on: # [push] push: branches: - master @@ -27,16 +27,16 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-devops-python-tools # ${{ hashFiles('**/requirements.txt') }} restore-keys: | - ${{ runner.os }}-pip- + ${{ runner.os }}-pip-devops-python-tools - name: build run: make - name: test diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index c03f7072e..1ae4728b7 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -27,7 +27,7 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: ubuntu:14.04 env: diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 714dbbaf7..b7c311aca 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -27,7 +27,7 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: ubuntu:16.04 env: diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index d3ab2752c..6de120ffa 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -27,7 +27,7 @@ on: # [push] jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ubuntu-latest container: ubuntu:18.04 env: From 3dc035d9a4abea14373d08fd1fc16d674de0a635 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 10:49:45 +0000 Subject: [PATCH 0929/2779] added mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/mac_10.15.yaml diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml new file mode 100644 index 000000000..e0b628e6a --- /dev/null +++ b/.github/workflows/mac_10.15.yaml @@ -0,0 +1,45 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Mac 10.15 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 10 30 * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: macos-10.15 + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v1 + with: + path: ~/Library/Caches/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: brew update + run: which brew && brew update || : + - name: build + run: make + - name: test + run: make test From b3422c46b7e612e33c0081da221856e8a18df875 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 11:30:06 +0000 Subject: [PATCH 0930/2779] renamed centos6.yaml to centos6.yaml.disabled --- .github/workflows/{centos6.yaml => centos6.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos6.yaml => centos6.yaml.disabled} (100%) diff --git a/.github/workflows/centos6.yaml b/.github/workflows/centos6.yaml.disabled similarity index 100% rename from .github/workflows/centos6.yaml rename to .github/workflows/centos6.yaml.disabled From b2d8552d966e133eee50463b272ee90f30635b56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 11:30:15 +0000 Subject: [PATCH 0931/2779] updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 54e939f7a..410e2f97f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ Hari Sekhon - DevOps Python Tools [![CI Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+9%22) [![CI Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+10%22) [![CI CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS%22) -[![CI CentOS 6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%206/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+6%22) [![CI CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+7%22) [![CI CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+CentOS+8%22) [![CI Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Fedora%22) From 58b244411de1721e89036a651b92b417bb997083 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 13:52:07 +0000 Subject: [PATCH 0932/2779] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 54d79dbe9..cb52cc8ed 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -26,7 +26,7 @@ Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead eg. by ~1900 tables the Hive version will overtake the Impala version and - for thousands of tables Impala actuallys runs ~1.5x slower than the Hive version overall + for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From 465c2fba6c9dc819f3ee0f31446445b4b10c7994 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 13:52:17 +0000 Subject: [PATCH 0933/2779] updated impala_tables_locations.py --- impala_tables_locations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/impala_tables_locations.py b/impala_tables_locations.py index 12c0a93f2..3be05d6f8 100755 --- a/impala_tables_locations.py +++ b/impala_tables_locations.py @@ -25,6 +25,8 @@ Impala is faster than Hive for the first ~1000 tables but then slows down so if you have a lot of tables I recommend you use the Hive version of this instead + eg. by ~1900 tables the Hive version will overtake the Impala version and + for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL From b349db6009ef6e7784df696443a115c1be40af8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 17:24:07 +0000 Subject: [PATCH 0934/2779] updated test_welcome.sh --- tests/test_welcome.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_welcome.sh b/tests/test_welcome.sh index 9deb7ad8a..37e3d59af 100755 --- a/tests/test_welcome.sh +++ b/tests/test_welcome.sh @@ -24,6 +24,9 @@ cd "$srcdir/.."; # Fedora doesn't have /var/log/wtmp if ! [ -f /var/log/wtmp ]; then + echo "/var/log/wtmp doesn't exist, touching..." + # assigned in utils.sh + # shellcheck disable=SC2154 $sudo touch /var/log/wtmp || : fi From 4aa023e5e599118ccc3bfa7d4fe0415974b4ab60 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 21:04:52 +0000 Subject: [PATCH 0935/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index e0b628e6a..a16b0731f 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -38,7 +38,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip-devops-python-tools - name: brew update - run: which brew && brew update || : + run: which brew && brew update || echo - name: build run: make - name: test From 6da2cd7b05d4aac47fc5becc6c752598f1756c9f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Feb 2020 21:05:26 +0000 Subject: [PATCH 0936/2779] updated mac.yaml --- .github/workflows/mac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 3632e946c..0c1358164 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -38,7 +38,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip-devops-python-tools - name: brew update - run: which brew && brew update || : + run: which brew && brew update || echo - name: build run: make - name: test From 6f87ad2da49c8200569c7021dbb6c12293a56182 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 10:30:32 +0000 Subject: [PATCH 0937/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 46e4a2ace..a85df1ca3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 46e4a2acef5c34fc17cdab9350192cb8ee5f6e18 +Subproject commit a85df1ca3a6202c409707a8d9e45e4cc6d2c8b3f From 1e8e410c0ab371777b01301ee9f723456e284d16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 10:30:32 +0000 Subject: [PATCH 0938/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b5a9c16a2..3100736a8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b5a9c16a2e05e5d6a645a7ccfd68de7ba1780955 +Subproject commit 3100736a87133e99e4beea098574c8280f610a0d From 6c5992b08311adc4383261d304bccf79fbf591ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 14:15:41 +0000 Subject: [PATCH 0939/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a85df1ca3..feddf4f12 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a85df1ca3a6202c409707a8d9e45e4cc6d2c8b3f +Subproject commit feddf4f12d344257fb37c8eb548bdad736c11512 From 68331800b8882c2f6d2e23c3fdd029d34edb4da5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 14:15:41 +0000 Subject: [PATCH 0940/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3100736a8..85117d262 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3100736a87133e99e4beea098574c8280f610a0d +Subproject commit 85117d2628bf56105d984c82c3aca8437140e12a From c02354c9a1a6ae2b48258916bc6551e7a19a1b4e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 16:24:00 +0000 Subject: [PATCH 0941/2779] updated impala_tables_metadata.py --- impala_tables_metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index cb52cc8ed..c81b9305d 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -19,6 +19,11 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes +Examples: + +./impala_tables_metadata.py --field Location ... +./impala_tables_metadata.py --field SerDe ... + Caveats: Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't From 0ea0eb2205fbdde8aef52d4cef45b68451679662 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 16:24:06 +0000 Subject: [PATCH 0942/2779] updated hive_tables_metadata.py --- hive_tables_metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 04b89e728..a77972b16 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -19,6 +19,11 @@ Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') for all tables in all databases, or only those matching given db / table regexes +Examples: + +./hive_tables_metadata.py --field Location ... +./hive_tables_metadata.py --field SerDe ... + Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 5113ad70582888c697109774a9cc6e5a49cebe8c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 16:24:20 +0000 Subject: [PATCH 0943/2779] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index c81b9305d..659d0244a 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -19,7 +19,7 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes -Examples: +Examples (fields are case sensitive): ./impala_tables_metadata.py --field Location ... ./impala_tables_metadata.py --field SerDe ... From 6dec5d3e4bae527c4136f0bb5096c196707820cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Feb 2020 16:24:23 +0000 Subject: [PATCH 0944/2779] updated hive_tables_metadata.py --- hive_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index a77972b16..4755f829e 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -19,7 +19,7 @@ Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') for all tables in all databases, or only those matching given db / table regexes -Examples: +Examples (fields are case sensitive): ./hive_tables_metadata.py --field Location ... ./hive_tables_metadata.py --field SerDe ... From 9f7973532b45a5ed4b44f03a1e814cd74bc55ecc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:25:47 +0000 Subject: [PATCH 0945/2779] updated hive_tables_metadata.py --- hive_tables_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 4755f829e..36afc9875 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -19,7 +19,7 @@ Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') for all tables in all databases, or only those matching given db / table regexes -Examples (fields are case sensitive): +Examples (fields are case sensitive regex and return N/A without match): ./hive_tables_metadata.py --field Location ... ./hive_tables_metadata.py --field SerDe ... @@ -99,7 +99,7 @@ def process_options(self): # discard last param query and construct our own based on the table DDL of cols def execute(self, conn, database, table, query): log.info("describing table '%s.%s'", database, table) - field = 'UNKNOWN' + field = 'N/A' with conn.cursor() as table_cursor: # doesn't support parameterized query quoting from dbapi spec #table_cursor.execute('use %(database)s', {'database': database}) From f70ef2d79210de1227b83c5b704db8d388cb33f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:25:52 +0000 Subject: [PATCH 0946/2779] updated impala_tables_metadata.py --- impala_tables_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 659d0244a..09f79c484 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -19,7 +19,7 @@ Connect to an Impala daemon and list the locations of all tables in all databases, or only those matching given db / table regexes -Examples (fields are case sensitive): +Examples (fields are case sensitive regex and return N/A without match): ./impala_tables_metadata.py --field Location ... ./impala_tables_metadata.py --field SerDe ... From 0c9ebd8520311a9de819ff1f5d5ad6908c9523ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:29:46 +0000 Subject: [PATCH 0947/2779] updated impala_tables_metadata.py --- impala_tables_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 09f79c484..ba8a66cbc 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -16,8 +16,8 @@ """ -Connect to an Impala daemon and list the locations of all tables in all databases, -or only those matching given db / table regexes +Connect to an Impala daemon and print the first matching DDL metadata field (eg. 'Location') +for each table in each database, or only those matching given db / table regexes Examples (fields are case sensitive regex and return N/A without match): From a12f51855eeeaf45c098cbe16e77459ed1198422 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:29:52 +0000 Subject: [PATCH 0948/2779] updated hive_tables_metadata.py --- hive_tables_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 36afc9875..886c3eed8 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -16,8 +16,8 @@ """ -Connect to HiveServer2 and print the matching DDL metadata field (eg. 'Location') -for all tables in all databases, or only those matching given db / table regexes +Connect to HiveServer2 and print the first matching DDL metadata field (eg. 'Location') +for each table in each database, or only those matching given db / table regexes Examples (fields are case sensitive regex and return N/A without match): From b678761256f268d0178a554cd9141ad3b5f7f914 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:30:33 +0000 Subject: [PATCH 0949/2779] updated impala_foreach_table.py --- impala_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 7b35b0174..a47d846cc 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -16,7 +16,7 @@ """ -Connect to an Impala daemon and execute a query for all tables in all databases, +Connect to an Impala daemon and execute a query for each table in each database, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: From a81722383dffea5eba41872d1efa5051e29dc811 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Feb 2020 11:30:38 +0000 Subject: [PATCH 0950/2779] updated hive_foreach_table.py --- hive_foreach_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 4ef224290..1b40f4434 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -16,7 +16,7 @@ """ -Connect to HiveServer2 and execute a query for all tables in all databases, +Connect to HiveServer2 and execute a query for each table in each database, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: From c642b3c6dcca537ddc6eb9d6982547703596e77d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Feb 2020 23:05:08 +0000 Subject: [PATCH 0951/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index feddf4f12..c07860544 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit feddf4f12d344257fb37c8eb548bdad736c11512 +Subproject commit c07860544f350a9fc1e63fbc799baf1bb8957459 From 8c9164729302e76c9ae382d1feac1e095f41427a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Feb 2020 23:05:08 +0000 Subject: [PATCH 0952/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 85117d262..34f4467de 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 85117d2628bf56105d984c82c3aca8437140e12a +Subproject commit 34f4467ded90be3770e9cb447de7cb715e640a8e From 6bf152fbee61b7d7730cee9516a2fa3cf3a7a09f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Feb 2020 23:25:53 +0000 Subject: [PATCH 0953/2779] updated rpm-packages-pip.txt --- setup/rpm-packages-pip.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt index db112cbc7..c956b3189 100644 --- a/setup/rpm-packages-pip.txt +++ b/setup/rpm-packages-pip.txt @@ -25,3 +25,5 @@ python2-psutil #python-flask #python-markupsafe #python2-bitarray + +python3-snappy From 10977a0645988311fc7327cfeba1689c66fadadf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 10:00:21 +0000 Subject: [PATCH 0954/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c07860544..40a3466bb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c07860544f350a9fc1e63fbc799baf1bb8957459 +Subproject commit 40a3466bbf7ad0c34fbb467f8f25757265ddf166 From b2632a3ab2020716399c6d47f4ad8e92b891d350 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 10:00:22 +0000 Subject: [PATCH 0955/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 34f4467de..d4df4ea2b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 34f4467ded90be3770e9cb447de7cb715e640a8e +Subproject commit d4df4ea2b8a9160ce990d1737eeeefaea378ed19 From 77a635e32db2ecdecf7627944f094b127f6e43c6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 11:04:19 +0000 Subject: [PATCH 0956/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d4df4ea2b..432f81312 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d4df4ea2b8a9160ce990d1737eeeefaea378ed19 +Subproject commit 432f813127a62295abc9e1b25b7b535288656dd6 From cc1c4bd3a594b637274e81a2d75c8f32dbfd7a34 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 11:50:54 +0000 Subject: [PATCH 0957/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 40a3466bb..14c1c4033 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 40a3466bbf7ad0c34fbb467f8f25757265ddf166 +Subproject commit 14c1c40335c2f1f86c4e43bfbd5b45c0b530d322 From 855d19342fc6bfade64d809ddc2f64a6bd8a228a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 11:50:55 +0000 Subject: [PATCH 0958/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 432f81312..6faf9c1c5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 432f813127a62295abc9e1b25b7b535288656dd6 +Subproject commit 6faf9c1c5b24ee26a782d4066b83656f8c3ff4ae From 26358ae940f758c9486759c7ed74b92ec9bacb24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:02:04 +0000 Subject: [PATCH 0959/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 14c1c4033..69316f67e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 14c1c40335c2f1f86c4e43bfbd5b45c0b530d322 +Subproject commit 69316f67e12349ec9f4d704758d9edd313d33a2a From 79d659e5b7ae03a2521cd980885af524d6295de5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:02:05 +0000 Subject: [PATCH 0960/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6faf9c1c5..76913aba1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6faf9c1c5b24ee26a782d4066b83656f8c3ff4ae +Subproject commit 76913aba121992e1bb59dfb7adf1888c361e2fa3 From fc69f4f2419fce5421b1f66b6130284e864dff69 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:57:47 +0000 Subject: [PATCH 0961/2779] added hive_tables_column_counts.py --- hive_tables_column_counts.py | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100755 hive_tables_column_counts.py diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py new file mode 100755 index 000000000..0f993e60a --- /dev/null +++ b/hive_tables_column_counts.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and count the number of columns for each table in each database, +or only those matching given db / table regexes + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + +class HiveTablesColumnCounts(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # not needed, here to suppress --query CLI option + self.database = None + self.table = None + self.ignore_errors = False + + # discarding last param query + def execute(self, conn, database, table, query): + column_count = 0 + log.info("describing table '%s.%s'", database, table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use `{}`'.format(database)) + # don't use desc here, Impala doesn't support it and would break subclass + column_cursor.execute('describe `{}`'.format(table)) + for _ in column_cursor: + #column = _[0] + #column_type = _[1] + column_count += 1 + print('{db}.{table}\t{column_count}'.format(db=database, table=table, column_count=column_count)) + + +if __name__ == '__main__': + HiveTablesColumnCounts().main() From f29c560c04e7fa442a431bd805e29cc3e9689b74 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:57:50 +0000 Subject: [PATCH 0962/2779] added impala_tables_column_counts.py --- impala_tables_column_counts.py | 80 ++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 impala_tables_column_counts.py diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py new file mode 100755 index 000000000..5f9e6d486 --- /dev/null +++ b/impala_tables_column_counts.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and find tables with columns containing only NULLs +for all tables in all databases, or only those matching given db / table regexes + +Connect to an Impala daemon and count the number of columns for each table in each database, +or only those matching given db / table regexes + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_column_counts import HiveTablesColumnCounts +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesColumnCounts(HiveTablesColumnCounts): + + def __init__(self): + # Python 2.x + super(ImpalaTablesColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesColumnCounts().main() From 7d93a76ea7e3954e6afaeb8638d8b5ac28aa0877 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 14:59:04 +0000 Subject: [PATCH 0963/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 410e2f97f..21d3c5383 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations + - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs From 40f72c104b78fd5e308163e246f8b0490bfd39d9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:01:22 +0000 Subject: [PATCH 0964/2779] updated hive_tables_column_counts.py --- hive_tables_column_counts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py index 0f993e60a..100984804 100755 --- a/hive_tables_column_counts.py +++ b/hive_tables_column_counts.py @@ -60,6 +60,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class HiveTablesColumnCounts(HiveForEachTable): def __init__(self): From 295ef72c050a8cf07cab50de27299165cec08303 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:01:29 +0000 Subject: [PATCH 0965/2779] updated hive_tables_list.py --- hive_tables_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_list.py b/hive_tables_list.py index b0c4f4702..07c415780 100755 --- a/hive_tables_list.py +++ b/hive_tables_list.py @@ -63,6 +63,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.1.0' + class HiveTablesList(HiveForEachTable): def __init__(self): From 7858272398f0d859f6f6e99bfdca198acadc0b4b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:02:06 +0000 Subject: [PATCH 0966/2779] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index a78e049f4..84c0cc92b 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -67,6 +67,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class HiveTablesNullColumns(HiveForEachTable): def __init__(self): From 6b99408da172da05f24f16770a0f4c22c9e3357d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:02:16 +0000 Subject: [PATCH 0967/2779] updated hive_tables_null_rows.py --- hive_tables_null_rows.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 78c42d15e..8368e3c02 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -64,6 +64,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class HiveTablesNullRows(HiveForEachTable): def __init__(self): From 0c3c130b6b405a040efd6e2df671b97b81eb91d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:02:32 +0000 Subject: [PATCH 0968/2779] updated hive_tables_row_counts_any_nulls.py --- hive_tables_row_counts_any_nulls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 0f7a5a94a..7266d0b1e 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -64,6 +64,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class HiveTablesRowsWithNulls(HiveForEachTable): def __init__(self): From 3638bf19c6ba10869e75bc3619f006755dee1077 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:02:54 +0000 Subject: [PATCH 0969/2779] updated impala_tables_row_counts.py --- impala_tables_row_counts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 47c104789..3d748ded2 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -62,6 +62,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class ImpalaTablesRowCounts(HiveTablesRowCounts): def __init__(self): From 8d5f177c00b4d18bbd9ac8411f6a0b62f8a9144e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:08:41 +0000 Subject: [PATCH 0970/2779] updated impala_tables_column_counts.py --- impala_tables_column_counts.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py index 5f9e6d486..2489e4e9c 100755 --- a/impala_tables_column_counts.py +++ b/impala_tables_column_counts.py @@ -16,9 +16,6 @@ """ -Connect to an Impala daemon and find tables with columns containing only NULLs -for all tables in all databases, or only those matching given db / table regexes - Connect to an Impala daemon and count the number of columns for each table in each database, or only those matching given db / table regexes From ddce6027a12431a7738920d1c55c0384a033146b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:10:21 +0000 Subject: [PATCH 0971/2779] added hive_tables_row_column_counts.py --- hive_tables_row_column_counts.py | 110 +++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100755 hive_tables_row_column_counts.py diff --git a/hive_tables_row_column_counts.py b/hive_tables_row_column_counts.py new file mode 100755 index 000000000..660fef904 --- /dev/null +++ b/hive_tables_row_column_counts.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to HiveServer2 and count the number of rows and columns for each table +in each database, or only those matching given db / table regexes + +Output format: + + .
+ +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveTablesRowColumnCounts(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesRowColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # not needed, here to suppress --query CLI option + self.database = None + self.table = None + self.ignore_errors = False + + # discarding last param query + def execute(self, conn, database, table, query): + row_count = None + column_count = 0 + log.info("describing table '%s.%s'", database, table) + with conn.cursor() as cursor: + # doesn't support parameterized query quoting from dbapi spec + #cursor.execute('use %(database)s', {'database': database}) + #cursor.execute('describe %(table)s', {'table': table}) + cursor.execute('use `{}`'.format(database)) + # don't use desc here, Impala doesn't support it and would break subclass + cursor.execute('describe `{}`'.format(table)) + for _ in cursor: + #column = _[0] + #column_type = _[1] + column_count += 1 + log.info("running SELECT COUNT(*) FROM `%s`.`%s`", database, table) + # doesn't support parameterized query quoting from dbapi spec + cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}`'.format(db=database, table=table)) + for result in cursor: + assert row_count is None + row_count = result[0] + print('{db}.{table}\t{row_count}\t{column_count}'\ + .format(db=database, + table=table, + row_count=row_count, + column_count=column_count)) + + +if __name__ == '__main__': + HiveTablesRowColumnCounts().main() From 5e865f04e6e606580bdbc5cc515e375da18c0753 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:10:25 +0000 Subject: [PATCH 0972/2779] added impala_tables_row_column_counts.py --- impala_tables_row_column_counts.py | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 impala_tables_row_column_counts.py diff --git a/impala_tables_row_column_counts.py b/impala_tables_row_column_counts.py new file mode 100755 index 000000000..b2e867734 --- /dev/null +++ b/impala_tables_row_column_counts.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Connect to an Impala daemon and count the number of rows and columns for each table +in each database, or only those matching given db / table regexes + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_row_column_counts import HiveTablesRowColumnCounts +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesRowColumnCounts(HiveTablesRowColumnCounts): + + def __init__(self): + # Python 2.x + super(ImpalaTablesRowColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesRowColumnCounts().main() From 794415305e756fb9b79f5fa23b84e143bb83cf84 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 15:16:50 +0000 Subject: [PATCH 0973/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 21d3c5383..9b159e02a 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ Environment variables are supported for convenience and also to hide credentials - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables + - ```hive_tables_row_column_counts.py``` / ```impala_tables_row_column_counts.py``` - outputs tables row and column counts. Useful for finding unusually big tables - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs From 9d111fa28667faca5c5dfd5f4906e9da399001ef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 16:40:40 +0000 Subject: [PATCH 0974/2779] fix for Python 3 behaviour change around string => bytes --- tests/test_anonymize.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index 8d67fcf36..6850e650b 100755 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -40,8 +40,12 @@ def run(): print('running anonymize tests using: {} {}'.format(anonymize, args)) cmd = [anonymize] + [_ for _ in args.split()] process = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE) + # encode as bytes for Python 3 :-/ + test_input = str.encode(test_input, 'utf-8') (stdout, _) = process.communicate(input=test_input) index = 0 + # convert bytes to string + stdout = stdout.decode("utf-8") for line in stdout.split('\n'): # pylint: disable=redefined-outer-name key = src_keys[index] _input = src[key] From 41f852c4d600a2551e9e2515ba757ae812fbb768 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 17:44:21 +0000 Subject: [PATCH 0975/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 69316f67e..37e99e112 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 69316f67e12349ec9f4d704758d9edd313d33a2a +Subproject commit 37e99e1120f420820972b312154b3b0b4fd0817c From 5149cfcecc8d0def68613ecae0616334a6748089 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 17:44:22 +0000 Subject: [PATCH 0976/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 76913aba1..dc632d05f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 76913aba121992e1bb59dfb7adf1888c361e2fa3 +Subproject commit dc632d05fe529d90b8e41de55ddfb0fe46267991 From 4b1497593f23f42fc8aacb82b4975ff223ac172f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Feb 2020 17:48:30 +0000 Subject: [PATCH 0977/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index dc632d05f..e1862f618 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit dc632d05fe529d90b8e41de55ddfb0fe46267991 +Subproject commit e1862f618101114f92c668ef8263bf1a4ce0e2db From ea71e676c92f62ebc7be49a643733e4f83f8ac56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:20:32 +0000 Subject: [PATCH 0978/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 37e99e112..35e2e9a7a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 37e99e1120f420820972b312154b3b0b4fd0817c +Subproject commit 35e2e9a7a5b90528fcebcfcd76c2f1c4c777c8d1 From 5ca5fb46a1258e1c9324d4a4cded63d5ef0de17d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:20:32 +0000 Subject: [PATCH 0979/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e1862f618..46e0dcf96 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e1862f618101114f92c668ef8263bf1a4ce0e2db +Subproject commit 46e0dcf961daae7351bcf5a570ad09aa132fa6b9 From 791a132e3327ff269399676df9a9cfa6e34c3b8c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:30:05 +0000 Subject: [PATCH 0980/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 46e0dcf96..16f89c774 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 46e0dcf961daae7351bcf5a570ad09aa132fa6b9 +Subproject commit 16f89c77475f05ce2cf24468d14b5f2b31f4e2cf From cb8b2a3ed4f20165d6c5e1324047371ba39577f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:32:58 +0000 Subject: [PATCH 0981/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 887df140f..2b4b72a4e 100755 --- a/Makefile +++ b/Makefile @@ -100,7 +100,7 @@ python: @bash-tools/python_pip_install.sh snakebite[kerberos] || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - #@if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi + @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi bash-tools/python_pip_install.sh "avro-python3" || : @# for impyla From ea0b3e9e446f179c6b34930885112b30c9cda3a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:46:28 +0000 Subject: [PATCH 0982/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 35e2e9a7a..fa52c2bef 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 35e2e9a7a5b90528fcebcfcd76c2f1c4c777c8d1 +Subproject commit fa52c2bef66dc0966a46a300f7a2325398850eba From e094d084550ee6155bdacaae8299c22150127e01 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:46:28 +0000 Subject: [PATCH 0983/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 16f89c774..e409fa0b5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 16f89c77475f05ce2cf24468d14b5f2b31f4e2cf +Subproject commit e409fa0b5a8434b534e341d98227ca3a8930004d From 8f9068bb16099adbe08c1a61523c4060a80c74a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 11:52:13 +0000 Subject: [PATCH 0984/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e409fa0b5..532ae5f8f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e409fa0b5a8434b534e341d98227ca3a8930004d +Subproject commit 532ae5f8fd36dd7c9e54d432bc537d271cc9002c From 725002ebcc5db351fa353675adbce25b1e4dc546 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 13:46:01 +0000 Subject: [PATCH 0985/2779] updated requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77613f9cb..696aa5de9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,8 @@ python-snappy==0.5 sasl==0.2.1 sh==1.12.14 # pulls in python-KrbV as a dependency which doesn't build on Mac any more -# moved to Makefile as best effort +# relies on python-krbV is unmaintained and unported to Python 3 +# - moved to Makefile as best effort #snakebite[kerberos]==2.11.0 snakebite==2.11.0 thrift-sasl==0.2.1 From 96bc66bd1a5697dbc5d6778bb86fe5febe5a0eea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 13:46:34 +0000 Subject: [PATCH 0986/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2b4b72a4e..f6bb403d4 100755 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ python: @#$(SUDO_PIP) pip install -r requirements.txt @PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt - @# python-krbV dependency doesn't build on Mac any more and is unmaintained + @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit @bash-tools/python_pip_install.sh snakebite[kerberos] || : From ed5b34473b21e675133659968e9b50e9a3af14b1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 15:00:13 +0000 Subject: [PATCH 0987/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fa52c2bef..35fccb2dc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fa52c2bef66dc0966a46a300f7a2325398850eba +Subproject commit 35fccb2dce79b470373adc20da840fd6bccc7386 From 70957696d220ddb689b2c108eda52ca7e91e45f2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2020 15:00:13 +0000 Subject: [PATCH 0988/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 532ae5f8f..d091e31a5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 532ae5f8fd36dd7c9e54d432bc537d271cc9002c +Subproject commit d091e31a557ac9c5da729e7ae21fe2cd98a31ab6 From 7c146709ce3199bf53c308f629faefb072f5313f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 Feb 2020 14:52:32 +0000 Subject: [PATCH 0989/2779] updated apk-packages-pip.txt --- setup/apk-packages-pip.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt index 88bcf4c3a..0ab2bc769 100644 --- a/setup/apk-packages-pip.txt +++ b/setup/apk-packages-pip.txt @@ -16,6 +16,7 @@ py-dicttoxml py-psutil py-pyldap -py2-jinja2 -py2-numpy -#py-flask +py3-jinja2 +py3-numpy +#py3-flask +py3-pygit2 From 7eb28b2b0430a5d41f143e7c84ca62f1c4ee085b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 13:15:48 +0000 Subject: [PATCH 0990/2779] updated README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9b159e02a..bcc685a6f 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ Hari Sekhon - DevOps Python Tools [![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) From 046e0798e7e8fc4addc6e253b109624b4a8bed53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 14:20:55 +0000 Subject: [PATCH 0991/2779] added bootstrap.sh --- setup/bootstrap.sh | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 setup/bootstrap.sh diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh new file mode 100755 index 000000000..c8b387861 --- /dev/null +++ b/setup/bootstrap.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-10-16 10:33:03 +0100 (Wed, 16 Oct 2019) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# Alpine / Wget: +# +# wget https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh && sh bootstrap.sh +# +# Curl: +# +# curl https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh + +set -eu +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(dirname "$0")" + +repo="https://github.com/HariSekhon/DevOps-Python-tools" + +directory="pytools" + +if [ "$(uname -s)" = Darwin ]; then + echo "Bootstrapping Mac" + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install | ruby +elif [ "$(uname -s)" = Linux ]; then + echo "Bootstrapping Linux" + if type apk >/dev/null 2>&1; then + apk --no-cache add bash git make + elif type apt-get >/dev/null 2>&1; then + apt-get update + apt-get install -y git make + elif type yum >/dev/null 2>&1; then + yum install -y git make + else + echo "Package Manager not found on Linux, cannot bootstrap" + exit 1 + fi +else + echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time" + exit 1 +fi + +if [ "${srcdir##*/}" = setup ]; then + cd "$srcdir/.." +elif [ -d "$directory" ]; then + cd pytools +else + git clone "$repo" "$directory" + cd "$directory" +fi + +make From d0fece87532923e62de1d2ad6182743fab159daa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 14:25:16 +0000 Subject: [PATCH 0992/2779] updated Makefile --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f6bb403d4..afe57b99c 100755 --- a/Makefile +++ b/Makefile @@ -15,17 +15,21 @@ # =================== # bootstrap commands: +# setup/bootstrap.sh +# +# OR +# # Alpine: # -# apk add --no-cache git $(MAKE) && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE) - +# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# # Debian / Ubuntu: # -# apt-get update && apt-get install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE) - +# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# # RHEL / CentOS: # -# yum install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE) +# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make # =================== From 05fe5cdb21386fc8b279cd9c0d149d9bc23d5230 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 15:21:03 +0000 Subject: [PATCH 0993/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 696aa5de9..28d238a6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ dicttoxml==1.7.4 #elasticsearch>=1.0.0,<2.0.0 # fails on requiring newer version of setuptools #Flask==0.10.1 -GitPython==2.1.14 +GitPython==2.1.15 happybase==1.0.0 humanize==0.5.1 impyla==0.16.0 From d0270aae2cdafbcf142a7d5092501e82d16171a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2020 15:45:11 +0000 Subject: [PATCH 0994/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bcc685a6f..b638ec10b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) -[![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) From 04417ce4292cb50a475e3d03091909f807733fd1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 10:02:29 +0000 Subject: [PATCH 0995/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 35fccb2dc..aded5fdee 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 35fccb2dce79b470373adc20da840fd6bccc7386 +Subproject commit aded5fdeea7705f90aa3e5b5f1f96ad65f144d4b From 47af104ca60efcee561e6f10efb7511b9fbce8c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 10:02:30 +0000 Subject: [PATCH 0996/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d091e31a5..432e7d1a0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d091e31a557ac9c5da729e7ae21fe2cd98a31ab6 +Subproject commit 432e7d1a0e348601829edda48119a02a0778d76c From e61c50b6a67df1e66f16e391a9223b61a1d8e795 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 10:13:33 +0000 Subject: [PATCH 0997/2779] updated impala_tables_column_counts.py --- impala_tables_column_counts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py index 2489e4e9c..b26b0ebcc 100755 --- a/impala_tables_column_counts.py +++ b/impala_tables_column_counts.py @@ -19,6 +19,10 @@ Connect to an Impala daemon and count the number of columns for each table in each database, or only those matching given db / table regexes +You can also get this from the schemas.csv output generated by impala_schemas_csv.py, eg. + + tail -n +2 impala_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr + Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From 58ebf363d27ea68511f9948f09209b6bb919aff7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 10:13:56 +0000 Subject: [PATCH 0998/2779] updated hive_tables_column_counts.py --- hive_tables_column_counts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py index 100984804..2902f23dd 100755 --- a/hive_tables_column_counts.py +++ b/hive_tables_column_counts.py @@ -19,6 +19,10 @@ Connect to HiveServer2 and count the number of columns for each table in each database, or only those matching given db / table regexes +You can also get this from the schemas.csv output generated by hive_schemas_csv.py, eg. + + tail -n +2 hive_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr + Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see From c4739bda418e7632b764c0229e36cb3667b94e0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 13:02:36 +0000 Subject: [PATCH 0999/2779] updated dockerhub_search.py --- dockerhub_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerhub_search.py b/dockerhub_search.py index cb226e59f..405981dab 100755 --- a/dockerhub_search.py +++ b/dockerhub_search.py @@ -56,7 +56,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6' +__version__ = '0.6.1' class DockerHubSearch(CLI): @@ -71,7 +71,7 @@ def __init__(self): self.quiet = False def add_options(self): - self.add_opt('-n', '--num', '--limit', default=50, + self.add_opt('-n', '--num', '--limit', default=50, type=int, help='Number of results to return (default: 50)') self.add_opt('-q', '--quiet', action='store_true', help='Output only the image names, one per line (useful for shell scripting)') From ec0d785cb5641fe9918dcabd1e76ffed67654f46 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Feb 2020 18:01:08 +0000 Subject: [PATCH 1000/2779] updated bootstrap.sh --- setup/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index c8b387861..a77c27e19 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -53,7 +53,7 @@ fi if [ "${srcdir##*/}" = setup ]; then cd "$srcdir/.." elif [ -d "$directory" ]; then - cd pytools + cd "$directory" else git clone "$repo" "$directory" cd "$directory" From 6286d1a796c2e17f8a08653ef8188418684d6395 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:12:00 +0000 Subject: [PATCH 1001/2779] added azure-pipelines.yml --- azure-pipelines.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..0517a2142 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,25 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: Sun Feb 23 19:02:10 2020 +0000 +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +# https://aka.ms/yaml + +trigger: +- master + +pool: + vmImage: 'ubuntu-18.04' + +- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make default test" + displayName: docker build From 60a1f34cd658acbbf15e1839f2918bf31dc119c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:15:39 +0000 Subject: [PATCH 1002/2779] updated azure-pipelines.yml --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0517a2142..5c1e82316 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,5 +21,6 @@ trigger: pool: vmImage: 'ubuntu-18.04' +steps: - script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make default test" displayName: docker build From a363ea6cc01c214d4e52df0362aef80f4ef17f2e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:42:23 +0000 Subject: [PATCH 1003/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index aded5fdee..16025a4be 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit aded5fdeea7705f90aa3e5b5f1f96ad65f144d4b +Subproject commit 16025a4bea20a14e7c03c0a836adec81c0e5969d From 4a6e52396f7935336b9552a5cc664d294dadd6d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:42:24 +0000 Subject: [PATCH 1004/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 432e7d1a0..573575e48 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 432e7d1a0e348601829edda48119a02a0778d76c +Subproject commit 573575e4893a4116e13fe74b92395376d5305f4c From ce957375c7c3dd1d8228486e6c598028c3efbd89 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 17:53:09 +0000 Subject: [PATCH 1005/2779] updated azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5c1e82316..f0ecbeb5c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,5 +22,5 @@ pool: vmImage: 'ubuntu-18.04' steps: -- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make default test" +- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make init && make default test" displayName: docker build From 845a016d403991a260e63a3a39784f14f23629d9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 18:01:47 +0000 Subject: [PATCH 1006/2779] added .appveyor.yml --- .appveyor.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..d3790490b --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,28 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://www.appveyor.com/docs/appveyor-yml/ + +image: Ubuntu + +install: +- sudo apt update +- sudo apt install -y git make +- make + +test_script: +- make test + +build: off From fee24cae6c2dc30e9dd87b683671e96695587701 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 21:21:28 +0000 Subject: [PATCH 1007/2779] added codefresh.yml --- codefresh.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 codefresh.yml diff --git a/codefresh.yml b/codefresh.yml new file mode 100644 index 000000000..ceff141da --- /dev/null +++ b/codefresh.yml @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 17:43:07 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +version: "1.0" +stages: + - "checkout" + - "build" +steps: + checkout: + type: "git-clone" + description: "Cloning main repository..." + repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}' + revision: "${{CF_REVISION}}" + stage: "checkout" + build: + title: Running docker image + type: freestyle + working_directory: '${{CF_REPO_NAME}}' + arguments: + image: 'ubuntu:18.04' + commands: + - apt update && apt install -y git make + - make + - make test From 5fbb7698009acf2ad3711dae831f30c8f7ab3259 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 21:25:50 +0000 Subject: [PATCH 1008/2779] added shippable.yml --- shippable.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 shippable.yml diff --git a/shippable.yml b/shippable.yml new file mode 100644 index 000000000..215035109 --- /dev/null +++ b/shippable.yml @@ -0,0 +1,35 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# http://docs.shippable.com/platform/workflow/config/ + +language: none + +branches: + only: + - master + +build: + ci: + - shippable_retry make + - make test + +integrations: + notifications: + - integrationName: email + type: email + on_success: never + on_failure: never + on_pull_request: never From e46b39b8e5c31b1ec1ddd9f8a9f7529d614eb977 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 21:30:24 +0000 Subject: [PATCH 1009/2779] added wercker.yml --- wercker.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 wercker.yml diff --git a/wercker.yml b/wercker.yml new file mode 100644 index 000000000..9c7eb0961 --- /dev/null +++ b/wercker.yml @@ -0,0 +1,30 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://devcenter.wercker.com/reference/wercker-yml/ + +box: debian + +build: + steps: + - script: + name: install git & make + code: apt-get update && apt-get install -y git make + - script: + name: build + code: make + - script: + name: test + code: make test From f38054379caa53345bde90db3c305cf0d197906e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 Feb 2020 21:41:16 +0000 Subject: [PATCH 1010/2779] added config.yml --- .circleci/config.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..41b107303 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,28 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-23 23:30:14 +0000 (Sun, 23 Feb 2020) +# Original: H1 2016 (Circle CI 1.x) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://circleci.com/docs/2.0/configuration-reference + +version: 2.1 +jobs: + build: + machine: + #image: ubuntu-1604:201903-01 + image: default + steps: + - checkout + - run: make + - run: make test From 879d29ad3d4986fea5e0b5079d9d15b0ebe0fad9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:11 +0000 Subject: [PATCH 1011/2779] updated azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f0ecbeb5c..1967d5718 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,5 +22,5 @@ pool: vmImage: 'ubuntu-18.04' steps: -- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make init && make default test" +- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make init && make ci test" displayName: docker build From 77e5c37f12b5b73e2adcc05cefbb0f19f57bc51d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:12 +0000 Subject: [PATCH 1012/2779] updated codefresh.yml --- codefresh.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codefresh.yml b/codefresh.yml index ceff141da..35dd12f02 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -32,5 +32,6 @@ steps: image: 'ubuntu:18.04' commands: - apt update && apt install -y git make - - make + - make init + - make ci - make test From 61c84eb91646386eccc35f9238514becae3fb4d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:12 +0000 Subject: [PATCH 1013/2779] updated shippable.yml --- shippable.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shippable.yml b/shippable.yml index 215035109..68a3c8ef1 100644 --- a/shippable.yml +++ b/shippable.yml @@ -23,7 +23,9 @@ branches: build: ci: - - shippable_retry make + #- shippable_retry make + - make init + - make ci - make test integrations: From 573204fff51c4d9ecb0ee3edad6ddedfdbaaccac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:13 +0000 Subject: [PATCH 1014/2779] updated wercker.yml --- wercker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wercker.yml b/wercker.yml index 9c7eb0961..c1037b720 100644 --- a/wercker.yml +++ b/wercker.yml @@ -22,9 +22,12 @@ build: - script: name: install git & make code: apt-get update && apt-get install -y git make + - script: + name: init + code: make init - script: name: build - code: make + code: make ci - script: name: test code: make test From 7b4833a04908a1c2ee382bcc53f4ba57f4144021 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:36:13 +0000 Subject: [PATCH 1015/2779] updated .circleci/config.yml --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 41b107303..78c5f45d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/bash-tools # # License: see accompanying Hari Sekhon LICENSE file # @@ -24,5 +24,6 @@ jobs: image: default steps: - checkout - - run: make + - run: make init + - run: make ci - run: make test From 06a9bc84f58fc709c17383bb6645eab080a519ed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:38:51 +0000 Subject: [PATCH 1016/2779] added bitbucket-pipelines.yml --- bitbucket-pipelines.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 bitbucket-pipelines.yml diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml new file mode 100644 index 000000000..2fd727146 --- /dev/null +++ b/bitbucket-pipelines.yml @@ -0,0 +1,26 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 17:08:57 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://confluence.atlassian.com/x/5Q4SMw +# Only use spaces to indent your .yml configuration. +# ----- +# You can specify a custom docker image from Docker Hub as your build environment. +image: atlassian/default-image:2 + +pipelines: + default: + - step: + script: + - apt update && apt install -y git make && make init && make ci test From 169393bf618628271ca4e0cafc18274c6f1673c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:57:01 +0000 Subject: [PATCH 1017/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 16025a4be..c2bea1feb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 16025a4bea20a14e7c03c0a836adec81c0e5969d +Subproject commit c2bea1feb574904153069cbddaf9ef87cbcfaa24 From db5443f31647b81e8221e5f12e12d4ba9c200228 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:57:02 +0000 Subject: [PATCH 1018/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 573575e48..2518c2c3f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 573575e4893a4116e13fe74b92395376d5305f4c +Subproject commit 2518c2c3f551c8063c6af3f2d9836a250728d4d2 From 8c8ed18c635896d4f9f0c1d848876d7ed1f7b25e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 10:58:08 +0000 Subject: [PATCH 1019/2779] Initial Bitbucket Pipelines configuration From dafbd7115e48934996274ec9bdbbb580f7517569 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Feb 2020 12:07:04 +0000 Subject: [PATCH 1020/2779] added .sonarcloud.properties --- .sonarcloud.properties | 1 + 1 file changed, 1 insertion(+) create mode 100644 .sonarcloud.properties diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 000000000..a4496869e --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1 @@ +sonar.host.url=https://sonarcloud.io From 443ad32cf1b2e33373c85466f205ae950b4c395f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 14:27:06 +0000 Subject: [PATCH 1021/2779] added .drone.yml --- .drone.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 000000000..4dd6036f6 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +kind: pipeline +type: docker +name: default + +steps: +- name: build + image: ubuntu:18.04 +# environment: +# DEBUG: 1 + commands: + - apt update -qq + - apt install -qy git make + - make init + - make ci + - make test + +trigger: + branch: + - master From 3a6d75a42e6ae4fe130f53b6ccb05c61617162f6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:28:30 +0000 Subject: [PATCH 1022/2779] updated README.md --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b638ec10b..3a5f95e27 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ Hari Sekhon - DevOps Python Tools ================================= -[![Build Status](https://travis-ci.org/HariSekhon/DevOps-Python-tools.svg?branch=master)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) @@ -14,6 +13,19 @@ Hari Sekhon - DevOps Python Tools [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) +[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) +[![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) +[![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) +[![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) +[![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) +[![Shippable](https://api.shippable.com/projects/5e52c63445c70f0007ff5144/badge?branch=master)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) +[![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) +[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) + +[![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/DevOps-Python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/DevOps-Python-tools/addon/pipelines/home#!/) +[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) + [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) From c8076a4579104485271dbc71bde8a4d30ec6ab4e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:29:02 +0000 Subject: [PATCH 1023/2779] sync'd CI builds --- .github/workflows/alpine.yaml | 20 ++++++-------------- .github/workflows/alpine_3.yaml | 20 ++++++-------------- .github/workflows/centos.yaml | 20 ++++++-------------- .github/workflows/centos6.yaml.disabled | 20 ++++++-------------- .github/workflows/centos7.yaml | 20 ++++++-------------- .github/workflows/centos8.yaml | 20 ++++++-------------- .github/workflows/debian.yaml | 21 ++++++--------------- .github/workflows/debian_10.yaml | 21 ++++++--------------- .github/workflows/debian_8.yaml | 21 ++++++--------------- .github/workflows/debian_9.yaml | 21 ++++++--------------- .github/workflows/fedora.yaml | 20 ++++++-------------- .github/workflows/ubuntu_14.04.yaml | 21 ++++++--------------- .github/workflows/ubuntu_16.04.yaml | 21 ++++++--------------- .github/workflows/ubuntu_18.04.yaml | 21 ++++++--------------- azure-pipelines.yml | 18 +++++++++++++++++- bitbucket-pipelines.yml | 2 +- codefresh.yml | 4 +++- shippable.yml | 2 ++ wercker.yml | 2 +- 19 files changed, 108 insertions(+), 207 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index eee5180b8..b4ea64cf0 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apk add --no-cache git make + run: apk add --no-cache git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 45ba0ee60..b9d1305a4 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apk add --no-cache git make + run: apk add --no-cache git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index b4bedd79d..13ff19170 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos6.yaml.disabled b/.github/workflows/centos6.yaml.disabled index eb053b667..6ec523536 100644 --- a/.github/workflows/centos6.yaml.disabled +++ b/.github/workflows/centos6.yaml.disabled @@ -38,20 +38,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 2b3567790..7ebef0745 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 807ea4d85..3868ce406 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index c0d32c539..9969fb0a1 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 24499f3ec..7980bae04 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index e2cad44dd..a7acf16cc 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index f2d84ffb8..e03e57161 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 735a91d72..1f7100849 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -36,20 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - yum install -y git make + run: yum install -y git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 1ae4728b7..f04e0bfda 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index b7c311aca..c122a9cab 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 6de120ffa..ff17f01cd 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -36,21 +36,12 @@ jobs: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make + run: apt-get update -qq && apt-get install -qy git make - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive - name: build - run: | - cd "/tmp/$repo" && - make + run: cd "/tmp/$repo" && make ci - name: test - run: | - cd "/tmp/$repo" && - make test + run: cd "/tmp/$repo" && make test diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1967d5718..cdf98c65a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,8 +19,24 @@ trigger: - master pool: + # there is no /dev/stderr on this azure build! + #vmImage: 'ubuntu-latest' + # Ubuntu 16.04 required for docker container support, looks like 18.04 works too vmImage: 'ubuntu-18.04' +# unprivileged container without sudo, cannot install dependencies +#container: ubuntu:18.04 + +# doesn't work due to lack of sudo +#steps: +#- script: sudo apt-get update && sudo apt-get install -y git make +# displayName: install git & make +#- script: make +# displayName: build +#- script: make test +# displayName: test + +# hacky workaround to Azure Pipelines limitations :-( steps: -- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update && apt install -y git make && make init && make ci test" +- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update -qq && apt install -qy git make && make init && make ci test" displayName: docker build diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 2fd727146..f44aee45d 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -23,4 +23,4 @@ pipelines: default: - step: script: - - apt update && apt install -y git make && make init && make ci test + - apt update -qq && apt install -qy git make && make init && make ci test diff --git a/codefresh.yml b/codefresh.yml index 35dd12f02..70990129d 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -13,6 +13,8 @@ # https://www.linkedin.com/in/harisekhon # +# https://codefresh.io/docs/docs/codefresh-yaml/ + version: "1.0" stages: - "checkout" @@ -31,7 +33,7 @@ steps: arguments: image: 'ubuntu:18.04' commands: - - apt update && apt install -y git make + - apt update -qq && apt install -qy git make - make init - make ci - make test diff --git a/shippable.yml b/shippable.yml index 68a3c8ef1..6bcd8a238 100644 --- a/shippable.yml +++ b/shippable.yml @@ -15,6 +15,8 @@ # http://docs.shippable.com/platform/workflow/config/ +# http://docs.shippable.com/ci/advancedOptions/environmentVariables/ + language: none branches: diff --git a/wercker.yml b/wercker.yml index c1037b720..18b14ed8f 100644 --- a/wercker.yml +++ b/wercker.yml @@ -21,7 +21,7 @@ build: steps: - script: name: install git & make - code: apt-get update && apt-get install -y git make + code: apt-get update -qq && apt-get install -qy git make - script: name: init code: make init From 65aafde9667cbbbada8d45234e49a23684befc71 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:29:18 +0000 Subject: [PATCH 1024/2779] sync'd CI builds --- .appveyor.yml | 4 ++-- .circleci/config.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index d3790490b..e367a94c7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -18,8 +18,8 @@ image: Ubuntu install: -- sudo apt update -- sudo apt install -y git make +- sudo apt update -qq +- sudo apt install -qy git make - make test_script: diff --git a/.circleci/config.yml b/.circleci/config.yml index 78c5f45d8..9c36ac51b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/bash-tools +# https://github.com/harisekhon/devops-python-tools # # License: see accompanying Hari Sekhon LICENSE file # From 410c54c0a7e6e7114be0d5d8c6c2860b7edd9004 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:29:35 +0000 Subject: [PATCH 1025/2779] added .gitlab-ci.yml --- .gitlab-ci.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..1613d1410 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,24 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: Sun Feb 23 19:02:10 2020 +0000 +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +# https://docs.gitlab.com/ee/ci/yaml/README.html + +image: ubuntu:18.04 + +job: + before_script: + - apt-get update -qq && apt-get install -yq git make + script: + - make init && make ci test From 3914bf32cf3901a8f1cfcd782b85f0720d776293 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:29:36 +0000 Subject: [PATCH 1026/2779] added .cirrus.yml --- .cirrus.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .cirrus.yml diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 000000000..3006fac0c --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,20 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 16:55:36 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +container: + image: ubuntu:18.04 + +task: + script: sudo apt update -qq && apt install -qy git make && make init && make ci test From fb08cdc53e2bda1d22405f60adfa4ce77e7bb346 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:43:42 +0000 Subject: [PATCH 1027/2779] updated . --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c36ac51b..7217d95a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,5 +25,5 @@ jobs: steps: - checkout - run: make init - - run: make ci + - run: make - run: make test From 29b85f7e14f7fce80ee24f5b02eaf907f266245b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:44:00 +0000 Subject: [PATCH 1028/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a5f95e27..2a5f68d6c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/DevOps-Python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/DevOps-Python-tools/addon/pipelines/home#!/) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/DevOps-Python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) From 462a073630d74b8f889f17cf038673d46627316b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 16:45:29 +0000 Subject: [PATCH 1029/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a5f68d6c..4c566f3e6 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/DevOps-Python-tools/addon/pipelines/home#!/) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) From 1cb7107f72059cd0cca4cd0fe131e1a9e184071e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 29 Feb 2020 18:29:21 +0000 Subject: [PATCH 1030/2779] fix for Python 3 --- crunch_accounting_csv_statement_converter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 477198dbf..9755b4435 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.3' +__version__ = '0.7.0' class CrunchAccountingCsvStatementConverter(CLI): @@ -157,7 +157,7 @@ def reverse_contents(filename): return tmp_filename def detect_columns(self, csvreader): - headers = csvreader.next() + headers = next(csvreader) if headers[0][0] == '{': log.error('JSON opening braces detected, not a CSV?') return False @@ -224,7 +224,7 @@ def validate_csvreader(csvreader, filename): log.error('non-alphanumeric / quote opening character detected in CSV') return None count += 1 - except csv.Error as _: + except csv.Error as _: log.warning('file %s, line %s: %s', filename, csvreader.line_num, _) return None if count == 0: From 8acf4d0f3955a7d3cac4d6f7903220f08c54ef4f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Mar 2020 17:45:05 +0000 Subject: [PATCH 1031/2779] added urlencode.py --- urlencode.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100755 urlencode.py diff --git a/urlencode.py b/urlencode.py new file mode 100755 index 000000000..c24b0b5c9 --- /dev/null +++ b/urlencode.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to url encode text from standard input or a text argument + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon import CLI + from harisekhon.utils import isPythonMinVersion +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +# pylint: disable=no-name-in-module,import-error +if isPythonMinVersion(3): + from urllib.parse import quote +else: + from urllib import quote + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +class URLEncode(CLI): + + def run(self): + if len(sys.argv) > 1: + for arg in sys.argv[1:]: + self.encode(arg) + else: + for line in sys.stdin: + line = line.rstrip('\n').rstrip('\r') + self.encode(line) + + @staticmethod + def encode(string): + #print(urllib.parse.quote(string)) + print(quote(string)) + +if __name__ == '__main__': + URLEncode().main() From 1f5bc87bced8cc7ee96aeda4045a47f3f9f259da Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Mar 2020 17:54:10 +0000 Subject: [PATCH 1032/2779] updated urlencode.py --- urlencode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urlencode.py b/urlencode.py index c24b0b5c9..0b7987367 100755 --- a/urlencode.py +++ b/urlencode.py @@ -41,9 +41,9 @@ # pylint: disable=no-name-in-module,import-error if isPythonMinVersion(3): - from urllib.parse import quote + from urllib.parse import quote_plus as quote else: - from urllib import quote + from urllib import quote_plus as quote __author__ = 'Hari Sekhon' __version__ = '0.1.0' From 7c50b20f2db0a9b640095687f0d2e1c903c5d34d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Mar 2020 18:04:55 +0000 Subject: [PATCH 1033/2779] updated urlencode.py --- urlencode.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/urlencode.py b/urlencode.py index 0b7987367..ea8ca96ae 100755 --- a/urlencode.py +++ b/urlencode.py @@ -48,6 +48,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.1.0' + class URLEncode(CLI): def run(self): @@ -64,5 +65,6 @@ def encode(string): #print(urllib.parse.quote(string)) print(quote(string)) + if __name__ == '__main__': URLEncode().main() From 831c413ae98db1cde53bb049d0a260a8403df481 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Mar 2020 18:09:36 +0000 Subject: [PATCH 1034/2779] added urldecode.py --- urldecode.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 urldecode.py diff --git a/urldecode.py b/urldecode.py new file mode 100755 index 000000000..d2b9d89a3 --- /dev/null +++ b/urldecode.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Tool to url decode text from standard input or a text argument + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon import CLI + from harisekhon.utils import isPythonMinVersion +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +# pylint: disable=no-name-in-module,import-error +if isPythonMinVersion(3): + from urllib.parse import unquote_plus as unquote +else: + from urllib import unquote_plus as unquote + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class URLDecode(CLI): + + def run(self): + if len(sys.argv) > 1: + for arg in sys.argv[1:]: + self.decode(arg) + else: + for line in sys.stdin: + line = line.rstrip('\n').rstrip('\r') + self.decode(line) + + @staticmethod + def decode(string): + print(unquote(string)) + + +if __name__ == '__main__': + URLDecode().main() From 974fe6a639d8262f7275bf86f7bbdf08cfb73ef8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Mar 2020 09:39:15 +0000 Subject: [PATCH 1035/2779] updated center.py --- center.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/center.py b/center.py index 752db7f6a..8ac287184 100755 --- a/center.py +++ b/center.py @@ -42,7 +42,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.4.1' class Center(CLI): @@ -53,6 +53,7 @@ def __init__(self): # super().__init__() self.re_bound = re.compile(r'(\b)') self.re_chars = re.compile(r'([^\s])(?!\s)') + self.timeout_default = None def add_options(self): self.add_opt('-w', '--width', default=80, type='int', metavar='', From ceedd3626c57274b6c0295521b09fec4522b1451 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Mar 2020 09:42:38 +0000 Subject: [PATCH 1036/2779] updated urlencode.py --- urlencode.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/urlencode.py b/urlencode.py index ea8ca96ae..756dee829 100755 --- a/urlencode.py +++ b/urlencode.py @@ -46,11 +46,18 @@ from urllib import quote_plus as quote __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' class URLEncode(CLI): + def __init__(self): + # Python 2.x + super(URLEncode, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = None + def run(self): if len(sys.argv) > 1: for arg in sys.argv[1:]: @@ -64,6 +71,7 @@ def run(self): def encode(string): #print(urllib.parse.quote(string)) print(quote(string)) + sys.stdout.flush() if __name__ == '__main__': From fc0c2c9ac46e835e7af69ea131c78e02127c628d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Mar 2020 09:42:51 +0000 Subject: [PATCH 1037/2779] updated urldecode.py --- urldecode.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/urldecode.py b/urldecode.py index d2b9d89a3..0f9d1e0f0 100755 --- a/urldecode.py +++ b/urldecode.py @@ -46,11 +46,18 @@ from urllib import unquote_plus as unquote __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' class URLDecode(CLI): + def __init__(self): + # Python 2.x + super(URLDecode, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = None + def run(self): if len(sys.argv) > 1: for arg in sys.argv[1:]: @@ -63,6 +70,7 @@ def run(self): @staticmethod def decode(string): print(unquote(string)) + sys.stdout.flush() if __name__ == '__main__': From 4c57c347923f4e9244c12da4fe3c819f4332c5f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Mar 2020 09:46:29 +0000 Subject: [PATCH 1038/2779] worked around stdin buffering to avoid Control-D char clashing --- urldecode.py | 7 ++++++- urlencode.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/urldecode.py b/urldecode.py index 0f9d1e0f0..4dd2242b7 100755 --- a/urldecode.py +++ b/urldecode.py @@ -63,7 +63,12 @@ def run(self): for arg in sys.argv[1:]: self.decode(arg) else: - for line in sys.stdin: + # buffered - Control-D char meshes with late output + #for line in sys.stdin: + while True: + line = sys.stdin.readline() + if not line: + break line = line.rstrip('\n').rstrip('\r') self.decode(line) diff --git a/urlencode.py b/urlencode.py index 756dee829..90d6ab5e6 100755 --- a/urlencode.py +++ b/urlencode.py @@ -63,7 +63,12 @@ def run(self): for arg in sys.argv[1:]: self.encode(arg) else: - for line in sys.stdin: + # buffered - Control-D char meshes with late output + #for line in sys.stdin: + while True: + line = sys.stdin.readline() + if not line: + break line = line.rstrip('\n').rstrip('\r') self.encode(line) From f03972770bc52cc3297dad025aa17d0dd65e2176 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 10:38:47 +0000 Subject: [PATCH 1039/2779] updated .cirrus.yml --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 3006fac0c..38752ca97 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -17,4 +17,4 @@ container: image: ubuntu:18.04 task: - script: sudo apt update -qq && apt install -qy git make && make init && make ci test + script: apt update -qq && apt install -qy git make && make init && make ci test From adb9a8071164581b4e1674a4ea3269c25a4e8ffa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:00:23 +0000 Subject: [PATCH 1040/2779] updated test_apache-drill.sh --- tests/test_apache-drill.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_apache-drill.sh b/tests/test_apache-drill.sh index c9a675207..fd336dc2c 100755 --- a/tests/test_apache-drill.sh +++ b/tests/test_apache-drill.sh @@ -82,6 +82,6 @@ test_apache_drill(){ echo } -startupwait 70 +startupwait 120 run_test_versions "Apache Drill" From 623aded98ac969659180ffa84857b92b62a65a37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:02:05 +0000 Subject: [PATCH 1041/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c2bea1feb..6d0113404 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c2bea1feb574904153069cbddaf9ef87cbcfaa24 +Subproject commit 6d0113404f799d1309a0a4ff868e88c9b33adf0c From 8081871fe5dc7a75872074d8c18b54c7ee6b696d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:02:07 +0000 Subject: [PATCH 1042/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2518c2c3f..2f193da9e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2518c2c3f551c8063c6af3f2d9836a250728d4d2 +Subproject commit 2f193da9ea9c3aa2af8285bc560490009d5bb17c From 28eed4dd8ab0f2f45d8ca1f2ccb316c3e904492b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:02:35 +0000 Subject: [PATCH 1043/2779] updated excluded.sh --- tests/excluded.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/excluded.sh b/tests/excluded.sh index 68f62c58f..a85efb228 100755 --- a/tests/excluded.sh +++ b/tests/excluded.sh @@ -23,7 +23,9 @@ set -eu isExcluded(){ local prog="$1" [[ "$prog" =~ ^\* ]] && return 0 - [[ "$prog" =~ spark_.*.py ]] && return 0 + [[ "$prog" =~ spark_.*\.py ]] && return 0 + [[ "$prog" =~ \.jy ]] && return 0 + [[ "$prog" =~ hdfs_find_replication_factor_1\.py ]] && return 0 # python-krbV doesn't build on Python 3 #[[ $prog =~ ipython-notebook ]] && return 0 # this external git check is expensive, skip it when in CI as using fresh git checkouts is_CI && return 1 From d992bcb82d44f65daea7dc59ae64d38df07131d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 16:04:12 +0000 Subject: [PATCH 1044/2779] updated help.sh --- tests/help.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/help.sh b/tests/help.sh index 7b4edd786..8666b576c 100755 --- a/tests/help.sh +++ b/tests/help.sh @@ -33,10 +33,8 @@ for x in ${@:-$(echo ./*.py 2>/dev/null)}; do echo; hr if [ $status = 0 ]; then [[ "$x" =~ ambari_blueprints.py$ ]] && continue - [[ "$x" =~ (hive|impala)_schemas_csv.py$ ]] && continue - [[ "$x" =~ (hive|impala)_foreach_table.py$ ]] && continue - [[ "$x" =~ (hive|impala)_tables_row_counts.py$ ]] && continue [[ "$x" =~ pythonpath.py$ ]] && continue + [[ "$x" =~ aws_s3_presign.py$ ]] && continue elif [ $status = 1 ]; then if [[ "$x" =~ hdfs_find_replication_factor_1.py$ ]] && ! python -c 'import krbV'; then # best effort, not available on Mac any more From bc398ab2af6ed8ab33f14146002fc06e519ddcc9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 17:12:25 +0000 Subject: [PATCH 1045/2779] updated welcome.py --- welcome.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/welcome.py b/welcome.py index 0d93b836f..bde562ac9 100755 --- a/welcome.py +++ b/welcome.py @@ -45,7 +45,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '2.0.2' +__version__ = '2.0.3' class Welcome(CLI): @@ -121,7 +121,7 @@ def print_welcome(self): print(msg) return try: - charmap = list(string.uppercase + string.lowercase + '@#$%^&*()') + charmap = list(string.ascii_uppercase + string.ascii_lowercase + '@#$%^&*()') # print '', # print('', end='') for i in range(0, len(msg)): From 3b8abb83bd19d965f71997b444062a3f4d0ce05c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 18:09:50 +0000 Subject: [PATCH 1046/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6d0113404..dd6b9b20b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6d0113404f799d1309a0a4ff868e88c9b33adf0c +Subproject commit dd6b9b20b52bd5c0daf9b3fec57b3ffef4f67ad3 From 92d0a3df6109a23cbba61ffa2206e2860d010fb2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 18:09:51 +0000 Subject: [PATCH 1047/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2f193da9e..4118be214 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2f193da9ea9c3aa2af8285bc560490009d5bb17c +Subproject commit 4118be214098593bc27e8411634dca5236b0e206 From 661e2d45d214b0915ee2d7e2b4c84ab7549614bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 18:10:17 +0000 Subject: [PATCH 1048/2779] updated all.sh --- tests/all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/all.sh b/tests/all.sh index 7aa1f8c75..b1e1957dc 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -29,7 +29,8 @@ section "Running PyTools ALL" # runs against . by default cd "$srcdir/.."; -bash-tools/check_all.sh +# has to be included so that isExcluded function is inherited +. bash-tools/check_all.sh #tests/test_yamllint.sh From 194fe41d5d03f3838b9b341a2441812cd7777a32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 20:52:11 +0000 Subject: [PATCH 1049/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index dd6b9b20b..c447bad9b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit dd6b9b20b52bd5c0daf9b3fec57b3ffef4f67ad3 +Subproject commit c447bad9bd012b61a363647338abd95e2f2a8ab4 From c80ae2a12a3319f8087aacbb5d7dcaa9c42a6f67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 20:52:11 +0000 Subject: [PATCH 1050/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4118be214..f226a567a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4118be214098593bc27e8411634dca5236b0e206 +Subproject commit f226a567aa86ecb8ce73dab1b75b038ae7b6e347 From 73c1f321b54593df42d2f324c06fa448fb120ae5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 20:55:54 +0000 Subject: [PATCH 1051/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c447bad9b..93c88e317 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c447bad9bd012b61a363647338abd95e2f2a8ab4 +Subproject commit 93c88e3177f22de869e02529390d7aff1b95dfb9 From 43af4dd3ef22f9adcd38eb533dae54646da65e14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 5 Mar 2020 20:55:54 +0000 Subject: [PATCH 1052/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f226a567a..fcf075f55 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f226a567aa86ecb8ce73dab1b75b038ae7b6e347 +Subproject commit fcf075f55e11856efb4982d3098e3eeada525310 From 5cb11a3f5290b22cd446074371af05f64de40d65 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:31:03 +0000 Subject: [PATCH 1053/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 93c88e317..266052396 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 93c88e3177f22de869e02529390d7aff1b95dfb9 +Subproject commit 2660523969ad17ad8864973a4c390afe6ccd266b From 84db42ec1eac9b1b1f94e0c4b070b96dba3a5ecd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:31:05 +0000 Subject: [PATCH 1054/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fcf075f55..37044fa45 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fcf075f55e11856efb4982d3098e3eeada525310 +Subproject commit 37044fa45bcd4aa11090c2bdee06f6e40225b879 From fc639e6718f55b083cb9ae0ed380ec79c695ba0d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:38:20 +0000 Subject: [PATCH 1055/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 266052396..43528a5b4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2660523969ad17ad8864973a4c390afe6ccd266b +Subproject commit 43528a5b4b1de2e2e2805ef0c5c6ebc203a0edd8 From 5dd80cd20d832cd9f349e315377504945afd46f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:38:39 +0000 Subject: [PATCH 1056/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 28d238a6e..299a1b6d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ sh==1.12.14 # relies on python-krbV is unmaintained and unported to Python 3 # - moved to Makefile as best effort #snakebite[kerberos]==2.11.0 -snakebite==2.11.0 +#snakebite==2.11.0 thrift-sasl==0.2.1 thrift==0.9.3 thriftpy==0.3.9 From 31317cddf9efffbe44050078a6f65b21171f5e55 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:39:08 +0000 Subject: [PATCH 1057/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index afe57b99c..7d93ca278 100755 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ python: @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit - @bash-tools/python_pip_install.sh snakebite[kerberos] || : + @bash-tools/setup/python_install_snakebite.sh # Python >= 3.4 - try but accept failure in case we're not on the right version of Python @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi From ed553cbe88238ba1e3d9a46074640437d37fdc02 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 10:41:13 +0000 Subject: [PATCH 1058/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7d93ca278..c3689b47e 100755 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ python: @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit - @bash-tools/setup/python_install_snakebite.sh + bash-tools/setup/python_install_snakebite.sh # Python >= 3.4 - try but accept failure in case we're not on the right version of Python @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi From 90f06ad0f247d8cf4f73ff64db5de5a3ea298a59 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 6 Mar 2020 16:07:17 +0000 Subject: [PATCH 1059/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c566f3e6..819795fee 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,9 @@ Environment variables are supported for convenience and also to hide credentials - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```json_to_xml.py``` - JSON to XML converter - ```xml_to_json.py``` - XML to JSON converter + - ```json_to_xml.py``` - JSON to XML converter + - ```json_to_yaml.py``` - JSON to YAML converter - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - see also ```validate_*.py``` further down for all these formats and more - [Ambari](https://hortonworks.com/apache/ambari/): From f7b2f23022cf42be528791396bfeceebda778a73 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 7 Mar 2020 19:16:39 +0000 Subject: [PATCH 1060/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 43528a5b4..af8183da2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 43528a5b4b1de2e2e2805ef0c5c6ebc203a0edd8 +Subproject commit af8183da22d16bbbceef97764f7d03fd078b0475 From d1ee03ae8afffd3955ed7cc1ec8edb80a65b1195 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 7 Mar 2020 19:16:39 +0000 Subject: [PATCH 1061/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 37044fa45..825ad2c93 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 37044fa45bcd4aa11090c2bdee06f6e40225b879 +Subproject commit 825ad2c93fc2733fab365557eb973d240b6678b7 From 264633e8b8583f22d8eaccc962c0b8454b6acf15 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 7 Mar 2020 22:24:09 +0000 Subject: [PATCH 1062/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index af8183da2..f62b996e0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit af8183da22d16bbbceef97764f7d03fd078b0475 +Subproject commit f62b996e03be15d3907aa612da0c8577dbe914c8 From 303eca20af7866a669728705ff375649a3c2530f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 7 Mar 2020 22:24:09 +0000 Subject: [PATCH 1063/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 825ad2c93..96b13e1c9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 825ad2c93fc2733fab365557eb973d240b6678b7 +Subproject commit 96b13e1c96789ce684cf865b57f9bd5324332bc0 From 52bfe12d65199431b13c35bd3af242593c2ebd9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 00:08:16 +0000 Subject: [PATCH 1064/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f62b996e0..34034522a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f62b996e03be15d3907aa612da0c8577dbe914c8 +Subproject commit 34034522a7e38cd7e41444fb21157a5f6c544625 From 2b6cf3bf30b270c7f5b1367b31ebd9e7575bace8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 00:08:16 +0000 Subject: [PATCH 1065/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 96b13e1c9..2b3bffa4b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 96b13e1c96789ce684cf865b57f9bd5324332bc0 +Subproject commit 2b3bffa4b91da25931deed6493f1915ac2aef861 From c5ccdf2c69d1010bba0b5f854c4f756f347fbf6e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 00:41:37 +0000 Subject: [PATCH 1066/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 34034522a..9cfc896b2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 34034522a7e38cd7e41444fb21157a5f6c544625 +Subproject commit 9cfc896b24982e442c2c7849c32c76589a56fc0c From 25afa9197cc7b7ec6fd28c307b30188adbc49cda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 00:41:37 +0000 Subject: [PATCH 1067/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2b3bffa4b..73ec58d48 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2b3bffa4b91da25931deed6493f1915ac2aef861 +Subproject commit 73ec58d48246f7c061227cf26dab9a6bb6a99601 From e8f03bd79da4b2c931c84ea52a143d5bc754cc1f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 09:35:20 +0000 Subject: [PATCH 1068/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9cfc896b2..690a0a4d2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9cfc896b24982e442c2c7849c32c76589a56fc0c +Subproject commit 690a0a4d2c25fca1f29bb87085e12d7c8b97aa5f From 4147702ca04d783e7760a90fc9b2d9f0b2002210 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 09:35:20 +0000 Subject: [PATCH 1069/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 73ec58d48..5bf3bbc40 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 73ec58d48246f7c061227cf26dab9a6bb6a99601 +Subproject commit 5bf3bbc408df3f9e181043796547a83a6d383d32 From 5e0ae9dd6dcfc385fdd168ae780afd78cb859401 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:13:59 +0000 Subject: [PATCH 1070/2779] updated README.md --- README.md | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 819795fee..3b11a8439 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,42 @@ Hari Sekhon - DevOps Python Tools ================================= -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) + +[![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) + +[![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) -[![Platform](https://img.shields.io/badge/platform-Linux%20%7C%20OS%20X-blue.svg)](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools) -[![DockerHub](https://img.shields.io/badge/docker-available-blue.svg)](https://hub.docker.com/r/harisekhon/pytools/) -[![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) +[![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Docker](https://img.shields.io/badge/container-Docker-blue?logo=docker)](https://hub.docker.com/r/harisekhon/github/) +[![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-github?label=DockerHub%20pulls&logo=docker)](https://hub.docker.com/r/harisekhon/github) [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) -[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) -[![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) -[![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) +[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) +[![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) +[![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://api.shippable.com/projects/5e52c63445c70f0007ff5144/badge?branch=master)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) +[![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/lib/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) +[![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) From 5b5fb2e0c59c618404f4fbf5928e992adbdf51ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:17:58 +0000 Subject: [PATCH 1071/2779] updated README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 3b11a8439..a1a1f11f7 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,12 @@ docker run harisekhon/pytools #### Automated Build from source ##### +installs git, make, pulls the repo and build the dependencies: +``` +curl https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh +``` + +or manually: ``` git clone https://github.com/harisekhon/devops-python-tools pytools cd pytools From 29c2d8ca5977ade7e96fe7be4d387d3b80a187a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:37:54 +0000 Subject: [PATCH 1072/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a1a1f11f7..bb5cf2ef2 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ Hari Sekhon - DevOps Python Tools ================================= [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) +[![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 02dd9ec9569447c4f202e78bd5fac1ba9f586f3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:40:02 +0000 Subject: [PATCH 1073/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 690a0a4d2..d65b95ff3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 690a0a4d2c25fca1f29bb87085e12d7c8b97aa5f +Subproject commit d65b95ff3393906e7e7923787af56a89818a3f21 From 24263453aa8d6df9a4c185d3c72d2948f610ed8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 20:40:03 +0000 Subject: [PATCH 1074/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5bf3bbc40..b38575728 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5bf3bbc408df3f9e181043796547a83a6d383d32 +Subproject commit b385757288c78ce4a9024b353573f71ad7700fc3 From 6f9f291f039758cbd17f53f763f714feb1e896c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 22:56:05 +0000 Subject: [PATCH 1075/2779] updated aws_users_last_used.py --- aws_users_last_used.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 2657f03f7..588a1d7af 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -103,10 +103,10 @@ def run(self): break log.info('waiting for credentials report') time.sleep(1) - try: - result = iam.get_credential_report() - except ClientError as _: - raise + #try: + result = iam.get_credential_report() + #except ClientError as _: + # raise csv_content = result['Content'] log.debug('%s', csv_content) filehandle = StringIO(unicode(csv_content)) From fde38fda835c8451fdddd1721f48e133ca630b0c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 8 Mar 2020 22:56:18 +0000 Subject: [PATCH 1076/2779] Update psutil from 4.3.0 to 5.7.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 299a1b6d4..35cc9f30b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ ldif3==3.2.2 #numpy==1.17.2 numpy==1.16.5 python-cson==1.0.9 -psutil==4.3.0 +psutil==5.7.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 #PyHive==0.6.1 From cdadedd3479681a224e79464758e8318723b81cf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 23:01:09 +0000 Subject: [PATCH 1077/2779] updated hive_tables_row_counts_any_nulls.py --- hive_tables_row_counts_any_nulls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 7266d0b1e..ce17ca99a 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -79,7 +79,7 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): + def execute(self, conn, database, table, _query): columns = [] log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: From dfcf2d1195ce62ec6709b1a835c546b4b071753f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 23:01:36 +0000 Subject: [PATCH 1078/2779] updated hive_tables_null_rows.py --- hive_tables_null_rows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 8368e3c02..7ab8b53dd 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -79,7 +79,7 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): + def execute(self, conn, database, table, _query): columns = [] log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: From 7546846094d89ea091016ea8a7bfecb2d1ae5d76 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Mar 2020 23:01:52 +0000 Subject: [PATCH 1079/2779] updated hive_tables_null_columns.py --- hive_tables_null_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 84c0cc92b..5874a1565 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -82,7 +82,7 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): + def execute(self, conn, database, table, _query): sum_part = '' columns = [] log.info("describing table '%s.%s'", database, table) From 793c03f03d59ea06c01672b47b8b13ceb3b9c311 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 18:04:49 +0000 Subject: [PATCH 1080/2779] added cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 294 ++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100755 cloudera_navigator_tables_used.py diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py new file mode 100755 index 000000000..dc647a528 --- /dev/null +++ b/cloudera_navigator_tables_used.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-09 11:35:47 +0000 (Mon, 09 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Processes Cloudera Navigator exported CSV logs to list the tables used (selected from) + +This allows you to see if you wasting time maintaining datasets nobody is using + +Handles more than naive filtering delimited column numbers which will miss many table and database names: + + 1. table/database name fields are often blank and need to be inferred from SQL queries field + 2. SQL queries often contain newlines which break the rows up + 3. multi-line SQL queries with commented out lines are stripped to avoid false positives of what is being used + +See cloudera_navigator_audit_download_logs.sh for a script to export these logs + +./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \ + navigator_audit_2020_hive.csv navigator_audit_2020_impala.csv + +Output - CSV format to stdout: + +database,table + +Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import csv +#import logging +import os +import re +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import CriticalError, log + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class ClouderaNavigatorTablesUsed(CLI): + + def __init__(self): + # Python 2.x + super(ClouderaNavigatorTablesUsed, self).__init__() + # Python 3.x + # super().__init__() + self.delimiter = None + self.quotechar = None + self.escapechar = None + self.data = {} + self.timeout_default = None + + def add_options(self): + super(ClouderaNavigatorTablesUsed, self).add_options() + # must set type to str otherwise csv module gives this error on Python 2.7: + # TypeError: "delimiter" must be string, not unicode + # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals' + # breaks this - might break in Python 3 if the impyla module doesn't fix behaviour + self.add_opt('-d', '--delimiter', default=',', type=str, help='Delimiter to use for outputting (default: ,)') + self.add_opt('-Q', '--quotechar', default='"', type=str, + help='Generate quoted CSV output (recommended, default is double quote \'"\')') + self.add_opt('-E', '--escapechar', help='Escape char if needed (for both reading and writing)') + + def process_options(self): + super(ClouderaNavigatorTablesUsed, self).process_options() + self.delimiter = self.get_opt('delimiter') + self.quotechar = self.get_opt('quotechar') + self.escapechar = self.get_opt('escapechar') + if not self.args: + self.usage('no CSV file argument given') + + def run(self): + quoting = csv.QUOTE_ALL + if self.quotechar == '': + quoting = csv.QUOTE_NONE + + fieldnames = ['database', 'table'] + csv_writer = csv.DictWriter(sys.stdout, + delimiter=self.delimiter, + quotechar=self.quotechar, + escapechar=self.escapechar, + quoting=quoting, + fieldnames=fieldnames) + for filename in self.args: + self.process_file(filename) + + csv_writer.writeheader() + for database in sorted(self.data): + for table in sorted(self.data[database]): + csv_writer.writerow({'database': database, + 'table': table}) + #if log.isEnabledFor(logging.DEBUG): + # sys.stdout.flush() + +# csv_header_indices.sh navigator_audit_2019_hive.csv +# 0 Timestamp +# 1 Username +# 2 "IP Address" +# 3 "Service Name" +# 4 Operation +# 5 Resource +# 6 Allowed +# 7 Impersonator +# 8 sub_operation +# 9 entity_id +# 10 stored_object_name +# 11 additional_info +# 12 collection_name +# 13 solr_version +# 14 operation_params +# 15 service +# 16 operation_text +# 17 url +# 18 operation_text +# 19 table_name +# 20 resource_path +# 21 database_name +# 22 object_type +# 23 Source +# 24 Destination +# 25 Permissions +# 26 "Delegation Token ID" +# 27 "Table Name" +# 28 Family +# 29 Qualifier +# 30 "Operation Text" +# 31 "Database Name" +# 32 "Table Name" +# 33 "Object Type" +# 34 "Resource Path" +# 35 "Usage Type" +# 36 "Operation Text" +# 37 "Query ID" +# 38 "Session ID" +# 39 Status +# 40 "Database Name" +# 41 "Table Name" +# 42 "Object Type" +# 43 Privilege + + # TODO: should really be refactored to be smaller simpler chunks of code + # XXX: this post processing is ugly as hell and probably brittle - YMMV + def process_file(self, filename): + re_select_from_table = re.compile(r'\bselect\b.+\bfrom\b(?:\s|\n)+([^\s,]+)', re.I | re.MULTILINE | re.DOTALL) + operations_to_ignore = [ + '', + 'HIVEREPLICATIONCOMMAND', + 'START', + 'STOP', + 'RESTART', + 'LOAD', + 'SWITCHDATABASE', + ] + with open(filename) as csvfile: + csv_reader = csv.reader(csvfile, delimiter=',', quotechar='"', escapechar='\\') + headers = csv_reader.next() + len_headers = len(headers) + # needed to ensure row joining works later on with number of fields left + assert len_headers == 44 + operation_index = 4 + table_index = 19 + database_index = 21 + sql_index = 36 + assert headers[operation_index] == 'Operation' + assert headers[table_index] == 'table_name' + assert headers[database_index] == 'database_name' + assert headers[sql_index] == 'Operation Text' + partial_row = [] + sql_decomment = self.sql_decomment + # more complicated than I wish it was - msg me if you know a simpler cleaner way + for row in csv_reader: + #log.debug('row = %s', row) + #try: + # various logic to handle rows broken on newlines inside SQL queries + len_row = len(row) + if len_row > len_headers: + #log.debug('collapsing fields in row: %s', row) + difference = len_row - len_headers + row[sql_index] = ','.join([sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index + difference:] + len_row = len(row) + #log.debug('collapsed row: %s', row) + #log.debug('row length: %s', len_row) + #log.debug('partial row length: %s', len(partial_row)) + if len_row == len_headers: + pass + elif len_row < len_headers: + log.debug('row (partial): %s', row) + if len_row + len(partial_row) == len_headers + 1: + #log.debug('length row + partial_row == header length, completing partial row') + #log.debug('partial_row = %s', partial_row) + #log.debug('row = %s', row) + # join first field to last field to complete SQL query + sql_fragment = sql_decomment(row[0]) + partial_row[-1] = partial_row[-1] + r'\n ' + sql_fragment + partial_row += row[1:] + #log.debug('partial_row = %s', partial_row) + elif partial_row: + #log.debug('partial_row: %s', partial_row) + #log.debug('row: %s', row) + # join next fragment of SQL query to incomplete last item containing the first part of SQL query + partial_row[-1] = partial_row[-1] + r'\n ' + r'\n '.join(row) + #log.debug('accumulated partial row: %s', partial_row) + elif len(partial_row) > len_headers: + raise CriticalError('len(partial_row) > len_headers - {} > {} for partial row: {}'\ + .format(len(partial_row), len_headers, partial_row)) + else: + partial_row = row + #log.debug('partial_row = %s', partial_row) + #log.debug('len partial row = %s', len(partial_row)) + #log.debug('len headers = %s', len_headers) + if len(partial_row) == len_headers: + # process accumulated row as normal + row = partial_row + partial_row = [] + #log.debug('accumulated completed row: %s', row) + else: + continue + elif partial_row: + raise CriticalError('incompleted partial row: {}'.format(partial_row)) + len_row = len(row) + if len_row != len_headers: + raise CriticalError('row items ({}) != header items ({}) for offending row: {}'\ + .format(len_row, len_headers, row)) + #log.debug(row) + database = row[21] + table = row[19] + if not table.strip(): + operation = row[4] + if operation == 'QUERY': + log.debug('table not found in row: %s', row) + query = row[36] + log.debug('trying to parse: %s', query) + match = re_select_from_table.search(query) + if match: + table = match.group(1) + if '.' in table: + (database, table) = table.split('.', 1) + else: + log.warning('failed to parse table from query: %s', query) + elif operation in operations_to_ignore: + continue + else: + log.debug('table not found in row and operation is not a query to parse: %s', row) + if not table and not database: + continue + table = table.lower() + database = database.lower() + self.data[database] = self.data.get(database, {}) + self.data[database][table] = 1 + #except IndexError as _: +# if log.isEnabledFor(logging.DEBUG): +# log.error('%s - offending line: %s', _, row) +# else: + # raise CriticalError('ERROR: %s - offending line: %s', _, row) + + @staticmethod + def sql_decomment(string): + return string.split('--')[0] + + +if __name__ == '__main__': + ClouderaNavigatorTablesUsed().main() From 6bdd602487bf01a75b8c290a51b7c151f2531a51 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:12:52 +0000 Subject: [PATCH 1081/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb5cf2ef2..c580598ce 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ Hari Sekhon - DevOps Python Tools ================================= +[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) -[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) From 0851091c709324e48ac749efb1975df65fdf788b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:12:59 +0000 Subject: [PATCH 1082/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d65b95ff3..40bb5733b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d65b95ff3393906e7e7923787af56a89818a3f21 +Subproject commit 40bb5733b5fef2ed71b86aac6be9a7235840f351 From 57848d6ed97ce0fc8e98532b8a6ed99bdb5dda13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:12:59 +0000 Subject: [PATCH 1083/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b38575728..f7ac21d1c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b385757288c78ce4a9024b353573f71ad7700fc3 +Subproject commit f7ac21d1c57d4e5b09b91007c9c9a0034e73fcb0 From 2c56c08c96db81be0b6be9065851227c42441f19 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:29:37 +0000 Subject: [PATCH 1084/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c580598ce..968e879fb 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ Hari Sekhon - DevOps Python Tools ================================= -[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) -[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) From e1f599fb13bf0b3ca07458cc3c9533ffd604b735 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:29:45 +0000 Subject: [PATCH 1085/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 40bb5733b..cfbe5cbfd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 40bb5733b5fef2ed71b86aac6be9a7235840f351 +Subproject commit cfbe5cbfd7b4f2edf749416a7893ba34ce3b9d4e From 8da549398810c4a20f167c44ac2f01d4eef47fed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 9 Mar 2020 22:29:46 +0000 Subject: [PATCH 1086/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f7ac21d1c..a2bc85bcd 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f7ac21d1c57d4e5b09b91007c9c9a0034e73fcb0 +Subproject commit a2bc85bcdfc6dd1d34f4a76ba5d1a31775888dcd From 609aac048bb5e8fdc5873a1cdb74a76dd7d2ee03 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 Mar 2020 13:37:31 +0000 Subject: [PATCH 1087/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cfbe5cbfd..a9eadd361 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cfbe5cbfd7b4f2edf749416a7893ba34ce3b9d4e +Subproject commit a9eadd3614474214c613859d4c7c9c279afa8b72 From f07be1e873f597318e7a5f793696fc9e92cb8262 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 Mar 2020 13:37:32 +0000 Subject: [PATCH 1088/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a2bc85bcd..ca05a3266 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a2bc85bcdfc6dd1d34f4a76ba5d1a31775888dcd +Subproject commit ca05a32663d6ac37fc28a5d765c97f2c12bf4f10 From dc51ee693093811574a0eec243a8c9a91477a13a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 Mar 2020 13:37:41 +0000 Subject: [PATCH 1089/2779] updated shippable.yml --- shippable.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/shippable.yml b/shippable.yml index 6bcd8a238..6b45ef91f 100644 --- a/shippable.yml +++ b/shippable.yml @@ -25,6 +25,13 @@ branches: build: ci: + # workaround to broken repos + # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB + # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 + # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed. + # bash-tools/Makefile.in:272: recipe for target 'apt-packages' failed + - rm -fv /etc/apt/sources.list.d/cassandra.sources.list* + - rm -fv /etc/apt/sources.list.d/yarn.list* #- shippable_retry make - make init - make ci From f03b5c80a2b2b171c77a2d3c1430ac90dd767b08 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 Mar 2020 13:37:46 +0000 Subject: [PATCH 1090/2779] updated .appveyor.yml --- .appveyor.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index e367a94c7..d53f810d5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -18,6 +18,22 @@ image: Ubuntu install: +# workaround for: +# Some packages could not be installed. This may mean that you have +# requested an impossible situation or if you are using the unstable +# distribution that some required packages have not yet been created +# or been moved out of Incoming. +# The following information may help to resolve the situation: +# +# The following packages have unmet dependencies: +# mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed +# E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. +# bash-tools/Makefile.in:272: recipe for target 'apt-packages' failed +# make[2]: *** [apt-packages] Error 123 +# make[2]: Leaving directory '/home/appveyor/projects/pylib' +# bash-tools/Makefile.in:212: recipe for target 'system-packages' failed +- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list || : +- sudo apt purge -y mssql-server || : - sudo apt update -qq - sudo apt install -qy git make - make From f7711e36c93fa53d2b839acca420c89ea2399c3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 10:47:03 +0000 Subject: [PATCH 1091/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a9eadd361..8a4e2e245 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a9eadd3614474214c613859d4c7c9c279afa8b72 +Subproject commit 8a4e2e24505539d1d5472755c631e5d8dfb265b7 From 876243882cd20b3d1a6fa8ae69be283db1760658 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 10:47:04 +0000 Subject: [PATCH 1092/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ca05a3266..b85f28276 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ca05a32663d6ac37fc28a5d765c97f2c12bf4f10 +Subproject commit b85f282766ad61a705984c06781e361154cb4cbf From 7e50bb844ed3d6c65843ed6f773a09ddbe5b438e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 10:48:35 +0000 Subject: [PATCH 1093/2779] updated .appveyor.yml .circleci/config.yml --- .appveyor.yml | 6 ++++-- .circleci/config.yml | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index d53f810d5..c1e4e06ac 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -32,8 +32,10 @@ install: # make[2]: *** [apt-packages] Error 123 # make[2]: Leaving directory '/home/appveyor/projects/pylib' # bash-tools/Makefile.in:212: recipe for target 'system-packages' failed -- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list || : -- sudo apt purge -y mssql-server || : +# +# adding "|| :" to the end of these commands causes them to be silently ignored! +- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list +- sudo apt purge -qy --allow-change-held-packages mssql-server - sudo apt update -qq - sudo apt install -qy git make - make diff --git a/.circleci/config.yml b/.circleci/config.yml index 7217d95a7..75c014403 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,9 +19,16 @@ version: 2.1 jobs: build: + # technically a docker image is a better choice than machine + # but we want to introduce some native environment variation + # between build systems in order to test the repo's build automation is robust machine: - #image: ubuntu-1604:201903-01 image: default + #image: ubuntu-1604:201903-01 + # set to an actual docker image when running locally using circle_ci_job.sh + # docker image must have git installed to do the checkout + # so using harisekhon/dev:ubuntu instead of base ubuntu image + #image: harisekhon/dev:ubuntu steps: - checkout - run: make init From 0a3c90d23de3d464ced58cb6fb160c0b5dc97a14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 14:22:24 +0000 Subject: [PATCH 1094/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 968e879fb..625d0aa42 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ Hari Sekhon - DevOps Python Tools [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) -[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) From 43c4a71b92a4d5de3a68f29e2f32d40c727f96b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 18:08:41 +0000 Subject: [PATCH 1095/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8a4e2e245..e7ad87b8f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8a4e2e24505539d1d5472755c631e5d8dfb265b7 +Subproject commit e7ad87b8f5d3c079014092a1eb468f6a7ef99cda From f11dd439e4490c2dc27b2a1f8be345a57411cba0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 18:08:42 +0000 Subject: [PATCH 1096/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b85f28276..4805995eb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b85f282766ad61a705984c06781e361154cb4cbf +Subproject commit 4805995ebe22c6edde2a10553f8c18f074665a3d From 686217a7a82fceb607ff8b3e1450d7a598f2e4ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 20:15:20 +0000 Subject: [PATCH 1097/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8a4e2e245..2fda7589e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8a4e2e24505539d1d5472755c631e5d8dfb265b7 +Subproject commit 2fda7589e4b4cf06b7b42467a1629f5e07812c5f From 66b376b2ecf16fbc26ab84c1131f695d3a82a36d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 20:15:20 +0000 Subject: [PATCH 1098/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b85f28276..46cab29f7 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b85f282766ad61a705984c06781e361154cb4cbf +Subproject commit 46cab29f7354502cccd295589396a7394115b969 From 2a3a358d60e8ee6cebeeeba3055c6c8afc74a623 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 23:13:28 +0000 Subject: [PATCH 1099/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2fda7589e..4a6277a0c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2fda7589e4b4cf06b7b42467a1629f5e07812c5f +Subproject commit 4a6277a0c21f817499d7181e0ad8e6f8f0a5699c From 70e5d480800809fcf0cca4e5086ff1b379bbdf13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 11 Mar 2020 23:13:28 +0000 Subject: [PATCH 1100/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 46cab29f7..c1093e128 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 46cab29f7354502cccd295589396a7394115b969 +Subproject commit c1093e128c061e7d62f92f0455dcbb7c70af5564 From 6520be12e036ea58456b038475eeb429108b1462 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 13:20:16 +0000 Subject: [PATCH 1101/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 625d0aa42..da235c81a 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STATUS.md) [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) From 5e4986fc837d7ecedafbdc1462e4609ed67fe578 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 17:21:32 +0000 Subject: [PATCH 1102/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4a6277a0c..f72e25d7e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4a6277a0c21f817499d7181e0ad8e6f8f0a5699c +Subproject commit f72e25d7eb699c38d248525c36197f6d99bb96d2 From e0afc03b5c6dcd29530e5bf2f812216efd2b442f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 17:21:33 +0000 Subject: [PATCH 1103/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c1093e128..7de1ce1dd 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c1093e128c061e7d62f92f0455dcbb7c70af5564 +Subproject commit 7de1ce1ddd165b8f91b285df5cce9a49fbba7b42 From 2556e5b9c72b4dc50285ca554b783306ec31e6e7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 17:41:14 +0000 Subject: [PATCH 1104/2779] updated requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 35cc9f30b..5df77ddb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,7 @@ ldif3==3.2.2 # Python 3.5+ #numpy==1.17.2 numpy==1.16.5 +psycopg2==2.8.4 python-cson==1.0.9 psutil==5.7.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools From aef4fbedcae40b67e6f4eed2a61e15ce308b705b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 12 Mar 2020 17:58:35 +0000 Subject: [PATCH 1105/2779] added lib/postgres_cli.py --- lib/postgres_cli.py | 83 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 lib/postgres_cli.py diff --git a/lib/postgres_cli.py b/lib/postgres_cli.py new file mode 100755 index 000000000..68864934f --- /dev/null +++ b/lib/postgres_cli.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-12 17:39:57 +0000 (Thu, 12 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import socket +import sys +import psycopg2 +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_host, validate_port + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class PostgreSQLCLI(CLI): + + def __init__(self): + # Python 2.x + super(PostgreSQLCLI, self).__init__() + # Python 3.x + # super().__init__() + self.name = ['PostgreSQL', 'Postgres', 'PG'] + self.host = None + self.port = None + self.default_host = socket.getfqdn() + self.default_port = 5432 + self.user = None + self.password = None + #self.ssl = False + self.verbose_default = 1 + self.timeout_default = None + + def add_options(self): + super(PostgreSQLCLI, self).add_options() + self.add_hostoption() + self.add_useroption() + self.add_opt('-d', '--database', help='Database to connect to') + + def process_options(self): + super(PostgreSQLCLI, self).process_options() + self.host = self.get_opt('host') + self.port = self.get_opt('port') + validate_host(self.host) + validate_port(self.port) + self.port = int(self.port) + self.user = self.user + self.password = self.password + #self.ssl = self.get_opt('ssl') + + def connect(self, database): + log.info('connecting to %s:%s database %s as user %s', self.host, self.port, database) + return psycopg2.connect(host=self.host, + port=self.port, + database=database, + user=self.user, + password=self.password) From e61f6e95c76be66d24e4844676c93e04131b4d9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 11:31:57 +0000 Subject: [PATCH 1106/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index da235c81a..56c669530 100644 --- a/README.md +++ b/README.md @@ -405,7 +405,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 100+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 200+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies * [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... From adc6d6a431c5e6cba8e489ecd84f960cccd527e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 12:04:09 +0000 Subject: [PATCH 1107/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f72e25d7e..50b4cf805 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f72e25d7eb699c38d248525c36197f6d99bb96d2 +Subproject commit 50b4cf805992083e8a675a41a979db56e5814901 From 0e4be2ff7f07c0580bbfa9a11d87ea540c778050 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 12:04:10 +0000 Subject: [PATCH 1108/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7de1ce1dd..01d8135a5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7de1ce1ddd165b8f91b285df5cce9a49fbba7b42 +Subproject commit 01d8135a535d15a52aadd3c9419130a32e75fb25 From 58e39ede54ad4c90ac214a935c37cb66106cd1dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 12:40:09 +0000 Subject: [PATCH 1109/2779] updated postgres_cli.py --- lib/postgres_cli.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/postgres_cli.py b/lib/postgres_cli.py index 68864934f..7ccd5b617 100755 --- a/lib/postgres_cli.py +++ b/lib/postgres_cli.py @@ -27,7 +27,7 @@ sys.path.append(libdir) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log, validate_host, validate_port + from harisekhon.utils import log, validate_host, validate_port, validate_user, validate_password from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -66,12 +66,14 @@ def add_options(self): def process_options(self): super(PostgreSQLCLI, self).process_options() self.host = self.get_opt('host') - self.port = self.get_opt('port') + self.host = self.get_opt('host') + self.user = self.get_opt('user') + self.password = self.get_opt('password') validate_host(self.host) validate_port(self.port) + validate_user(self.user) + validate_password(self.password) self.port = int(self.port) - self.user = self.user - self.password = self.password #self.ssl = self.get_opt('ssl') def connect(self, database): From 9293a92f99f82d59c630e4b6736f9770640b3381 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 13:06:35 +0000 Subject: [PATCH 1110/2779] updated requirements.txt --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5df77ddb1..9187ac1c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,9 @@ ldif3==3.2.2 # Python 3.5+ #numpy==1.17.2 numpy==1.16.5 -psycopg2==2.8.4 +# requires pg_config to build from source +#psycopg2==2.8.4 +psycopg2-binary==2.8.4 python-cson==1.0.9 psutil==5.7.0 # parquet support in pyarrow is weaker, gone back to using parquet-tools From 0026f3fcc09728faf6e88a4894ca19b6a7f364ad Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 13:16:20 +0000 Subject: [PATCH 1111/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 50b4cf805..e6161d191 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 50b4cf805992083e8a675a41a979db56e5814901 +Subproject commit e6161d19183fa3e9e71613c523add85ba35cfb06 From 341958ffa970c5889c6341fc7303ecf08a227576 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 13:16:20 +0000 Subject: [PATCH 1112/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 01d8135a5..1439449f0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 01d8135a535d15a52aadd3c9419130a32e75fb25 +Subproject commit 1439449f000f62886b5e69387f789a6be9fce08c From 4a21e96e579dca84b4944fd53823126af6c427bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 13 Mar 2020 21:39:07 +0000 Subject: [PATCH 1113/2779] added .buildkite/pipeline.yml --- .buildkite/pipeline.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 000000000..25a2cdc35 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,37 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-13 21:10:39 +0000 (Fri, 13 Mar 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# BuildKite Pipeline +# +# add this command to the UI and it will read the rest of the steps from here: +# +# - command: buildkite-agent pipeline upload + +steps: + - command: ./install_packages.sh make + label: install make + timeout: 10 + - wait + - command: make init + label: init + timeout: 2 + - wait + - command: make ci + label: build + timeout: 60 + - wait + - command: make test + label: test + timeout: 120 From 01c9a656fca176738f9112030697b8c8b98aac43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:39:30 +0000 Subject: [PATCH 1114/2779] updated pipeline.yml --- .buildkite/pipeline.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 25a2cdc35..c8bfbe881 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -20,9 +20,22 @@ # - command: buildkite-agent pipeline upload steps: - - command: ./install_packages.sh make + - command: | + type make 2>/dev/null || + if type apk 2>/dev/null; then + apk add --no-cache --no-progress make + # apt is /usr/bin/apt + # Unable to locate an executable at "/Users/hari/.sdkman/candidates/java/current/bin/apt" (-1) + elif type apt-get 2>/dev/null; then + apt-get update -q && + apt-get install -qy make + elif type yum 2>/dev/null; then + rpm -q make || yum install -y make + elif type brew 2>/dev/null; then + brew install make + fi label: install make - timeout: 10 + timeout: 20 # brew can take 10 mins just to do a brew update - wait - command: make init label: init From 8f628028196b1cd6100e58a1338ea27bf32ef7f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:48:13 +0000 Subject: [PATCH 1115/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 56c669530..5bd9249db 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Hari Sekhon - DevOps Python Tools [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) +[![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 1f13192c74407d58c590e81f834ed9dc76e4faa2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:49:08 +0000 Subject: [PATCH 1116/2779] updated apk-packages-dev.txt --- setup/apk-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt index 67e6f895d..efef4f8a5 100644 --- a/setup/apk-packages-dev.txt +++ b/setup/apk-packages-dev.txt @@ -14,6 +14,7 @@ # ============================================================================ # openldap-dev +postgresql-dev snappy-dev # installed by bash-tools submodule now From d1eebf208ed61a1b8186bdb1c4266e631b37bc5a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:51:52 +0000 Subject: [PATCH 1117/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index 6dd8fa477..2537f9e1a 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -17,6 +17,7 @@ gcc-c++ # needed to build python-krbV and cloudera/thrift_sasl cyrus-sasl-devel krb5-devel +libpq-devel # postgres pg_config openldap-devel openssl-devel From 77f8db87e2a110bc40b9bff095252469271a7162 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:52:28 +0000 Subject: [PATCH 1118/2779] updated rpm-packages-pip.txt --- setup/rpm-packages-pip.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt index c956b3189..1a879d376 100644 --- a/setup/rpm-packages-pip.txt +++ b/setup/rpm-packages-pip.txt @@ -26,4 +26,7 @@ python2-psutil #python-markupsafe #python2-bitarray +python2-psycopg2 +python3-psycopg2 + python3-snappy From 6eb2369a6668d3dab74e2d26c9810a9eef1b69a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:54:01 +0000 Subject: [PATCH 1119/2779] updated deb-packages-dev.txt --- setup/deb-packages-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index a2e311a6b..ae1a456ce 100644 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -14,6 +14,7 @@ # ============================================================================ # libldap2-dev +libpq-dev # postgres pg_config # needed to build python-snappy for avro module libsnappy-dev From 1383a2b084071abc025290a9b552cc5f1bf46595 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:54:58 +0000 Subject: [PATCH 1120/2779] updated deb-packages-pip.txt --- setup/deb-packages-pip.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt index 75196cfb9..8c6215957 100644 --- a/setup/deb-packages-pip.txt +++ b/setup/deb-packages-pip.txt @@ -24,6 +24,7 @@ python-ldap python-ldif3 python-numpy python-psutil +python-psycopg2 python-sh python-snappy python-thrift From 64ec4628ed120a102c4b4a72fcf949180c83e9b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:55:44 +0000 Subject: [PATCH 1121/2779] updated apk-packages-pip.txt --- setup/apk-packages-pip.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt index 0ab2bc769..01130d497 100644 --- a/setup/apk-packages-pip.txt +++ b/setup/apk-packages-pip.txt @@ -20,3 +20,4 @@ py3-jinja2 py3-numpy #py3-flask py3-pygit2 +py3-psycopg2 From ced5c3ace5bdba2906a063502487a1ad361c7e91 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:57:29 +0000 Subject: [PATCH 1122/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e6161d191..7b9ecd187 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e6161d19183fa3e9e71613c523add85ba35cfb06 +Subproject commit 7b9ecd187bf79e02f996915c4a606bbfa9da19e8 From 17983db85203d0badd94a5f430b82644904d1c8a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 14 Mar 2020 23:57:29 +0000 Subject: [PATCH 1123/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1439449f0..fe836adba 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1439449f000f62886b5e69387f789a6be9fce08c +Subproject commit fe836adba0c49c3a0b3316318653200469dcb6e1 From ff41881b970f837bca5592782aed6359c3406653 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Mar 2020 16:15:01 +0000 Subject: [PATCH 1124/2779] updated validate_json.py --- validate_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_json.py b/validate_json.py index 03cc0aa7a..1d83b8b04 100755 --- a/validate_json.py +++ b/validate_json.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.11.1' +__version__ = '0.11.2' class JsonValidatorTool(CLI): @@ -210,7 +210,7 @@ def check_json(self, content): return True self.failed = True if not self.passthru: - die(self.self.invalid_json_msg_single_quotes) + die(self.invalid_json_msg_single_quotes) else: log.debug('not valid json') if self.rewind_check_multirecord_json(): From 73e6506c2c87aa88c6ef0db9b0078bc054e2d9b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 17 Mar 2020 21:29:51 +0000 Subject: [PATCH 1125/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 345 ++++++++++++++++++------------ 1 file changed, 210 insertions(+), 135 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index dc647a528..fee6feb28 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -16,7 +16,7 @@ """ -Processes Cloudera Navigator exported CSV logs to list the tables used (selected from) +Processes Cloudera Navigator API exported CSV logs to list the tables used (selected from) This allows you to see if you wasting time maintaining datasets nobody is using @@ -25,15 +25,16 @@ 1. table/database name fields are often blank and need to be inferred from SQL queries field 2. SQL queries often contain newlines which break the rows up 3. multi-line SQL queries with commented out lines are stripped to avoid false positives of what is being used + 4. optionally ignore users by regex, matching user or kerberos principal to omit ETL service account -See cloudera_navigator_audit_download_logs.sh for a script to export these logs +See cloudera_navigator_audit_logs_download.sh for a script to export these logs -./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \ +./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \\ navigator_audit_2020_hive.csv navigator_audit_2020_impala.csv -Output - CSV format to stdout: +Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison): -database,table +"database","table" Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 @@ -45,7 +46,7 @@ #from __future__ import unicode_literals import csv -#import logging +import logging import os import re import sys @@ -56,7 +57,7 @@ sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import CriticalError, log + from harisekhon.utils import CriticalError, log, validate_regex from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -65,7 +66,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' class ClouderaNavigatorTablesUsed(CLI): @@ -78,8 +79,31 @@ def __init__(self): self.delimiter = None self.quotechar = None self.escapechar = None - self.data = {} + #self.data = {} + self.indicies = {} + self.len_headers = None + self.operations_to_ignore = [ + '', + 'HIVEREPLICATIONCOMMAND', + 'START', + 'STOP', + 'RESTART', + 'LOAD', + 'SWITCHDATABASE', + 'USE', + ] self.timeout_default = None + self.table_regex = r'[\w\.`]+' + self.re_table = re.compile(self.table_regex) + self.re_select_from_table = re.compile(r'\bSELECT\b.+\bFROM\b(?:\s|\n)+({table_regex})'\ + .format(table_regex=self.table_regex), \ + re.I | re.MULTILINE | re.DOTALL) + self.re_ignore = re.compile(r'\b(?:SHOW|DESCRIBE|USE|REFRESH|INVALIDATE\S+METADATA|GET_TABLES|GET_SCHEMAS)\b',\ + re.I | re.MULTILINE | re.DOTALL) + # 2020-01-31T20:45:59.000Z + self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') + self.re_ignored_users = None + self.csv_writer = None def add_options(self): super(ClouderaNavigatorTablesUsed, self).add_options() @@ -91,12 +115,20 @@ def add_options(self): self.add_opt('-Q', '--quotechar', default='"', type=str, help='Generate quoted CSV output (recommended, default is double quote \'"\')') self.add_opt('-E', '--escapechar', help='Escape char if needed (for both reading and writing)') + self.add_opt('-i', '--ignore-users', help='Users to ignore, comma separated regex values') def process_options(self): super(ClouderaNavigatorTablesUsed, self).process_options() self.delimiter = self.get_opt('delimiter') self.quotechar = self.get_opt('quotechar') self.escapechar = self.get_opt('escapechar') + ignore_users = self.get_opt('ignore_users') + if ignore_users: + ignored_users = ignore_users.split(',') + for username in ignored_users: + validate_regex(username, 'ignored user') + # account for kerberized names - user, user@domain.com or user/host@domain.com + self.re_ignored_users = re.compile('^' + '|'.join(ignored_users) + '(?:@|/|$)', re.I) if not self.args: self.usage('no CSV file argument given') @@ -106,23 +138,30 @@ def run(self): quoting = csv.QUOTE_NONE fieldnames = ['database', 'table'] - csv_writer = csv.DictWriter(sys.stdout, - delimiter=self.delimiter, - quotechar=self.quotechar, - escapechar=self.escapechar, - quoting=quoting, - fieldnames=fieldnames) + self.csv_writer = csv.DictWriter(sys.stdout, + delimiter=self.delimiter, + quotechar=self.quotechar, + escapechar=self.escapechar, + quoting=quoting, + fieldnames=fieldnames) for filename in self.args: - self.process_file(filename) + with open(filename, 'rU') as filehandle: + self.process_file(filehandle) - csv_writer.writeheader() - for database in sorted(self.data): - for table in sorted(self.data[database]): - csv_writer.writerow({'database': database, - 'table': table}) + #csv_writer.writeheader() + #for database in sorted(self.data): + # for table in sorted(self.data[database]): + # csv_writer.writerow({'database': database, + # 'table': table}) #if log.isEnabledFor(logging.DEBUG): # sys.stdout.flush() +# Navigator API Audit log output is a mess with duplicate columns and different naming conventions, +# eg. identical SQL in fields 18 and 36 +# table and database names in fields 19+21 vs 40+41 +# all duplicates with different header names +# +# # same result for navigator_audit_2019_impala.csv # csv_header_indices.sh navigator_audit_2019_hive.csv # 0 Timestamp # 1 Username @@ -169,125 +208,161 @@ def run(self): # 42 "Object Type" # 43 Privilege - # TODO: should really be refactored to be smaller simpler chunks of code - # XXX: this post processing is ugly as hell and probably brittle - YMMV - def process_file(self, filename): - re_select_from_table = re.compile(r'\bselect\b.+\bfrom\b(?:\s|\n)+([^\s,]+)', re.I | re.MULTILINE | re.DOTALL) - operations_to_ignore = [ - '', - 'HIVEREPLICATIONCOMMAND', - 'START', - 'STOP', - 'RESTART', - 'LOAD', - 'SWITCHDATABASE', - ] - with open(filename) as csvfile: - csv_reader = csv.reader(csvfile, delimiter=',', quotechar='"', escapechar='\\') - headers = csv_reader.next() - len_headers = len(headers) - # needed to ensure row joining works later on with number of fields left - assert len_headers == 44 - operation_index = 4 - table_index = 19 - database_index = 21 - sql_index = 36 - assert headers[operation_index] == 'Operation' - assert headers[table_index] == 'table_name' - assert headers[database_index] == 'database_name' - assert headers[sql_index] == 'Operation Text' - partial_row = [] - sql_decomment = self.sql_decomment - # more complicated than I wish it was - msg me if you know a simpler cleaner way - for row in csv_reader: - #log.debug('row = %s', row) - #try: - # various logic to handle rows broken on newlines inside SQL queries - len_row = len(row) - if len_row > len_headers: - #log.debug('collapsing fields in row: %s', row) - difference = len_row - len_headers - row[sql_index] = ','.join([sql_decomment(_) for _ in row[sql_index:difference]]) - row = row[:sql_index] + row[sql_index + difference:] - len_row = len(row) - #log.debug('collapsed row: %s', row) - #log.debug('row length: %s', len_row) - #log.debug('partial row length: %s', len(partial_row)) - if len_row == len_headers: + def process_file(self, filehandle): + csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') + headers = csv_reader.next() + self.len_headers = len(headers) + # needed to ensure row joining works later on with number of fields left + assert self.len_headers == 44 + user_index = 1 + operation_index = 4 + resource_index = 5 + table_index = 19 + database_index = 21 + # fields 18 and 36 are identical SQL - need both to collapse rows later + sql_index = 18 + sql_index2 = 36 + assert headers[user_index] == 'Username' + assert headers[operation_index] == 'Operation' + assert headers[resource_index] == 'Resource' + assert headers[table_index] == 'table_name' + assert headers[database_index] == 'database_name' + assert headers[sql_index] == 'operation_text' + assert headers[sql_index2] == 'Operation Text' + self.indicies = { + 'user_index': user_index, + 'operation_index': operation_index, + 'resource_index': resource_index, + 'table_index': table_index, + 'database_index': database_index, + 'sql_index': sql_index, + 'sql_index2': sql_index2, # needed for collapsing rows inflated by SQL fragmentation + } + self.process_rows(csv_reader) + + # logic to reconstruct rows because the Navigator API breaks the record format + # with newlines in SQL coming out literally and fragmenting the records + def process_rows(self, csv_reader): + last_row = [] + for current_row in csv_reader: + if not current_row: + continue + if self.re_timestamp.match(current_row[0]): + row = last_row + last_row = current_row + else: + last_row += current_row + continue + if not row: + continue + self.process_row(row) + self.process_row(last_row) + + def process_row(self, row): + log.debug('row = %s', row) + len_row = len(row) + if len_row > self.len_headers: + row = self.collapse_sql_fields(row=row) + len_row = len(row) + if len_row != self.len_headers: + raise CriticalError('row items ({}) != header items ({}) for offending row: {}'\ + .format(len_row, self.len_headers, row)) + (database, table) = self.parse_table(row) + self.output(row=row, database=database, table=table) + + def parse_table(self, row): + #log.debug(row) + user = row[self.indicies['user_index']] + # 'hari.sekhon' in 'hari.sekhon@somedomain.com' in kerberos + if self.re_ignored_users and self.re_ignored_users.match(user): + log.debug('skipping row for ignored user %s: %s', user, row) + return (None, None) + database = row[self.indicies['database_index']].strip() + table = row[self.indicies['table_index']].strip() + if not database or not table or not self.re_table.match('{}.{}'.format(database, table)): + #log.info('table not found in fields for row: %s', row) + operation = row[self.indicies['operation_index']] + if operation in self.operations_to_ignore: + return (None, None) + elif operation == 'QUERY': + query = row[self.indicies['sql_index']] + # cheaper than re_ignore to pre-filter + if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'): + return (None, None) + (database, table) = self.get_db_table_from_resource(row) + if database and table: pass - elif len_row < len_headers: - log.debug('row (partial): %s', row) - if len_row + len(partial_row) == len_headers + 1: - #log.debug('length row + partial_row == header length, completing partial row') - #log.debug('partial_row = %s', partial_row) - #log.debug('row = %s', row) - # join first field to last field to complete SQL query - sql_fragment = sql_decomment(row[0]) - partial_row[-1] = partial_row[-1] + r'\n ' + sql_fragment - partial_row += row[1:] - #log.debug('partial_row = %s', partial_row) - elif partial_row: - #log.debug('partial_row: %s', partial_row) - #log.debug('row: %s', row) - # join next fragment of SQL query to incomplete last item containing the first part of SQL query - partial_row[-1] = partial_row[-1] + r'\n ' + r'\n '.join(row) - #log.debug('accumulated partial row: %s', partial_row) - elif len(partial_row) > len_headers: - raise CriticalError('len(partial_row) > len_headers - {} > {} for partial row: {}'\ - .format(len(partial_row), len_headers, partial_row)) - else: - partial_row = row - #log.debug('partial_row = %s', partial_row) - #log.debug('len partial row = %s', len(partial_row)) - #log.debug('len headers = %s', len_headers) - if len(partial_row) == len_headers: - # process accumulated row as normal - row = partial_row - partial_row = [] - #log.debug('accumulated completed row: %s', row) - else: - continue - elif partial_row: - raise CriticalError('incompleted partial row: {}'.format(partial_row)) - len_row = len(row) - if len_row != len_headers: - raise CriticalError('row items ({}) != header items ({}) for offending row: {}'\ - .format(len_row, len_headers, row)) - #log.debug(row) - database = row[21] - table = row[19] - if not table.strip(): - operation = row[4] - if operation == 'QUERY': - log.debug('table not found in row: %s', row) - query = row[36] - log.debug('trying to parse: %s', query) - match = re_select_from_table.search(query) - if match: - table = match.group(1) - if '.' in table: - (database, table) = table.split('.', 1) - else: - log.warning('failed to parse table from query: %s', query) - elif operation in operations_to_ignore: - continue + else: + log.debug('database/table not found in row: %s', row) + log.debug('trying to parse: %s', query) + match = self.re_select_from_table.search(query) + if match: + table = match.group(1) + if '.' in table: + (database, table) = table.split('.', 1) + # could use .search but all these seem to be at beginning + elif self.re_ignore.match(query): + return (None, None) else: - log.debug('table not found in row and operation is not a query to parse: %s', row) - if not table and not database: - continue - table = table.lower() - database = database.lower() - self.data[database] = self.data.get(database, {}) - self.data[database][table] = 1 - #except IndexError as _: -# if log.isEnabledFor(logging.DEBUG): -# log.error('%s - offending line: %s', _, row) -# else: - # raise CriticalError('ERROR: %s - offending line: %s', _, row) + log.warning('failed to parse database/table from query: %s', query) + return (None, None) + else: + log.debug('database/table not found in row and operation is not a query to parse: %s', row) + return (None, None) + if not table and not database: + return (None, None) + table = table.lower().strip('`') + database = database.lower().strip('`') + if ' ' in table: + raise CriticalError('table "{}"'.format(table)) + if ' ' in database: + raise CriticalError('database "{}"'.format(database)) + return (database, table) + + def get_db_table_from_resource(self, row): + database = None + table = None + resource = row[self.indicies['resource_index']] + if resource: + # database:table in Resource field + (database, table) = resource.split(':', 1) + return (database, table) + + def output(self, row, database, table): + if not self.re_table.match('{}.{}'.format(database, table)): + log.warning('%s.%s does not match table regex', database, table) + return + #self.data[database] = self.data.get(database, {}) + #self.data[database][table] = 1 + if table and not database: + log.info('got table but not database for row: %s', row) + if database and not table: + log.info('got database but not table for row: %s', row) + if not table and not database: + return + self.csv_writer.writerow({'database': database, 'table': table}) + if log.isEnabledFor(logging.DEBUG): + sys.stdout.flush() + + def collapse_sql_fields(self, row): + sql_index = self.indicies['sql_index'] + sql_index2 = self.indicies['sql_index2'] + len_row = len(row) + if len_row > self.len_headers: + log.debug('collapsing fields in row: %s', row) + difference = len_row - self.len_headers + row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index+difference:] + row[sql_index2] = ','.join([self.sql_decomment(_) for _ in row[sql_index2:difference]]) + row = row[:sql_index2] + row[sql_index2+difference:] + log.debug('collapsed row: %s', row) + else: + log.debug('not collapsing row: %s', row) + return row @staticmethod def sql_decomment(string): - return string.split('--')[0] + return string.split('--')[0].strip() if __name__ == '__main__': From 33af4d3b176df8500852a2b096b3647e7428bbfb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Mar 2020 14:13:12 +0000 Subject: [PATCH 1126/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 39 ++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index fee6feb28..2918ce8b3 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -16,16 +16,20 @@ """ -Processes Cloudera Navigator API exported CSV logs to list the tables used (selected from) +Processes Cloudera Navigator API exported CSV logs to list the tables used (SELECT'ed from) -This allows you to see if you wasting time maintaining datasets nobody is using +This allows you to see if you're wasting time maintaining datasets nobody is using Handles more than naive filtering delimited column numbers which will miss many table and database names: 1. table/database name fields are often blank and need to be inferred from SQL queries field - 2. SQL queries often contain newlines which break the rows up - 3. multi-line SQL queries with commented out lines are stripped to avoid false positives of what is being used - 4. optionally ignore users by regex, matching user or kerberos principal to omit ETL service account + 2. SQL queries often contain newlines which break the rows up - these are recombined in to single records + 3. multi-line SQL queries have comments stripped out to avoid false positives of what is being used + 4. where table/database field aren't available, also checks resource field for suitable contents as another heuristic + to determine database and table name before parsing SQL which is a last resort + 5. optionally ignore selected users by regex + - matches user or kerberos principal + - eg. to omit ETL service account from skewing data access results See cloudera_navigator_audit_logs_download.sh for a script to export these logs @@ -57,7 +61,7 @@ sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import CriticalError, log, validate_regex + from harisekhon.utils import CriticalError, log, validate_regex, isInt from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -128,7 +132,7 @@ def process_options(self): for username in ignored_users: validate_regex(username, 'ignored user') # account for kerberized names - user, user@domain.com or user/host@domain.com - self.re_ignored_users = re.compile('^' + '|'.join(ignored_users) + '(?:@|/|$)', re.I) + self.re_ignored_users = re.compile('^(?:' + '|'.join(ignored_users) + ')(?:[@/]|$)', re.I) if not self.args: self.usage('no CSV file argument given') @@ -247,6 +251,7 @@ def process_rows(self, csv_reader): for current_row in csv_reader: if not current_row: continue + # originally did this by counting fields but SQL fragmentation generates extra fields if self.re_timestamp.match(current_row[0]): row = last_row last_row = current_row @@ -259,6 +264,8 @@ def process_rows(self, csv_reader): self.process_row(last_row) def process_row(self, row): + if not row: + return log.debug('row = %s', row) len_row = len(row) if len_row > self.len_headers: @@ -314,16 +321,20 @@ def parse_table(self, row): table = table.lower().strip('`') database = database.lower().strip('`') if ' ' in table: - raise CriticalError('table "{}"'.format(table)) + raise CriticalError('table \'{}\' has spaces - parsing error for row: {}'.format(table, row)) if ' ' in database: - raise CriticalError('database "{}"'.format(database)) + raise CriticalError('database \'{}\' has spaces - parsing error for row: {}'.format(database, row)) + if table == 'null': + raise CriticalError('table == null - parsing error for row: {}'.format(row)) return (database, table) def get_db_table_from_resource(self, row): database = None table = None resource = row[self.indicies['resource_index']] - if resource: + if resource and \ + ':' in resource and \ + 'null:null' not in resource: # database:table in Resource field (database, table) = resource.split(':', 1) return (database, table) @@ -350,7 +361,13 @@ def collapse_sql_fields(self, row): len_row = len(row) if len_row > self.len_headers: log.debug('collapsing fields in row: %s', row) - difference = len_row - self.len_headers + # divide by 2 to account for this having been done twice in duplicated SQL operational text + difference = (len_row - self.len_headers) / 2 + # slice indicies must be integers + if not isInt(difference): + raise CriticalError("difference in field length '{}' is not an integer for row: {}"\ + .format(difference, row)) + difference = int(difference) row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) row = row[:sql_index] + row[sql_index+difference:] row[sql_index2] = ','.join([self.sql_decomment(_) for _ in row[sql_index2:difference]]) From 87d6ba94dfc191266e02948824c47c8dafba36a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 18 Mar 2020 16:35:35 +0000 Subject: [PATCH 1127/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 966142672..53b219274 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -491,6 +491,9 @@ dest[141]="Failed to open HDFS file hdfs:///user//warehouse/ Date: Wed, 18 Mar 2020 16:36:09 +0000 Subject: [PATCH 1128/2779] added API token stripping --- anonymize.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/anonymize.py b/anonymize.py index 94beaf3f6..8e84d0c5f 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.10' +__version__ = '0.10.11' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -377,6 +377,9 @@ def __init__(self): sep=arg_sep, pass_word_phrase=pass_word_phrase, pw=password_quoted), + 'password4': r'([\.-]?(?:api-?)?token{sep}){pw}'\ + .format(sep=arg_sep, + pw=password_quoted), 'ip': r'(?', 'password2': r'\1:', 'password3': r'\1\2', + 'password4': r'\1', 'ip': r'/', 'ip2': r'', 'ip3': r'', From 98f0914624d46b3619ce0cde6cab8465e2033ec6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 10:36:29 +0000 Subject: [PATCH 1129/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5bd9249db..542df1894 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Hari Sekhon - DevOps Python Tools [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![All Code](https://img.shields.io/badge/all%20code-26k-lightgrey)](https://github.com/HariSekhon/DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) From 3ff8c1371d1935a10e60a3443202f7964eeaa7ad Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 10:40:39 +0000 Subject: [PATCH 1130/2779] updated README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 542df1894..680feb726 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,10 @@ Hari Sekhon - DevOps Python Tools [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) + +[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey)](https://github.com/HariSekhon/DevOps-Python-tools) [![PyUp](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/shield.svg)](https://pyup.io/account/repos/github/HariSekhon/DevOps-Python-tools/) [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) From 0c99607be2e13c8245536910054be6ffb3c1d97a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 12:07:55 +0000 Subject: [PATCH 1131/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 84 ++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 2918ce8b3..eb1cee773 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -23,9 +23,10 @@ Handles more than naive filtering delimited column numbers which will miss many table and database names: 1. table/database name fields are often blank and need to be inferred from SQL queries field + (currently limited to 'SELECT ... FROM ...' because JOINs are often complicated by use of table aliases) 2. SQL queries often contain newlines which break the rows up - these are recombined in to single records 3. multi-line SQL queries have comments stripped out to avoid false positives of what is being used - 4. where table/database field aren't available, also checks resource field for suitable contents as another heuristic + 4. where table/database field aren't available, also checks if inferrable from resource field to determine database and table name before parsing SQL which is a last resort 5. optionally ignore selected users by regex - matches user or kerberos principal @@ -41,6 +42,7 @@ "database","table" Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 +(but may require ongoing tweaks depending on quirks in your data set or changes in the API / logs) """ @@ -61,7 +63,7 @@ sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import CriticalError, log, validate_regex, isInt + from harisekhon.utils import log, validate_regex, isInt from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -99,6 +101,7 @@ def __init__(self): self.timeout_default = None self.table_regex = r'[\w\.`]+' self.re_table = re.compile(self.table_regex) + # doesn't handle JOINs because SQL pros usually use table aliases self.re_select_from_table = re.compile(r'\bSELECT\b.+\bFROM\b(?:\s|\n)+({table_regex})'\ .format(table_regex=self.table_regex), \ re.I | re.MULTILINE | re.DOTALL) @@ -221,18 +224,29 @@ def process_file(self, filehandle): user_index = 1 operation_index = 4 resource_index = 5 - table_index = 19 - database_index = 21 + object_index = 22 # used by collapse_sql_fields to check if SQL was split, do not change to index 33! + # -- + # with massive queries taking the latter 2 is more likely to succeed, + # possibly because there is a rare and subtle issue in collapse_sql_fields + #table_index = 19 + #database_index = 21 + # or + table_index = 41 + database_index = 40 + # -- # fields 18 and 36 are identical SQL - need both to collapse rows later sql_index = 18 sql_index2 = 36 + #assert headers[table_index] == 'table_name' # index 19 + #assert headers[database_index] == 'database_name' # index 21 + assert headers[table_index] == 'Table Name' # index 41 + assert headers[database_index] == 'Database Name' # index 40 assert headers[user_index] == 'Username' assert headers[operation_index] == 'Operation' assert headers[resource_index] == 'Resource' - assert headers[table_index] == 'table_name' - assert headers[database_index] == 'database_name' assert headers[sql_index] == 'operation_text' assert headers[sql_index2] == 'Operation Text' + assert headers[object_index] == 'object_type' self.indicies = { 'user_index': user_index, 'operation_index': operation_index, @@ -241,6 +255,7 @@ def process_file(self, filehandle): 'database_index': database_index, 'sql_index': sql_index, 'sql_index2': sql_index2, # needed for collapsing rows inflated by SQL fragmentation + 'object_index': object_index } self.process_rows(csv_reader) @@ -249,6 +264,7 @@ def process_file(self, filehandle): def process_rows(self, csv_reader): last_row = [] for current_row in csv_reader: + #log.debug('current row = %s', current_row) if not current_row: continue # originally did this by counting fields but SQL fragmentation generates extra fields @@ -266,13 +282,14 @@ def process_rows(self, csv_reader): def process_row(self, row): if not row: return - log.debug('row = %s', row) + log.debug('processing row = %s', row) len_row = len(row) + log.debug('row len = %s', len_row) if len_row > self.len_headers: row = self.collapse_sql_fields(row=row) len_row = len(row) if len_row != self.len_headers: - raise CriticalError('row items ({}) != header items ({}) for offending row: {}'\ + raise AssertionError('row items ({}) != header items ({}) for offending row: {}'\ .format(len_row, self.len_headers, row)) (database, table) = self.parse_table(row) self.output(row=row, database=database, table=table) @@ -318,14 +335,18 @@ def parse_table(self, row): return (None, None) if not table and not database: return (None, None) - table = table.lower().strip('`') - database = database.lower().strip('`') - if ' ' in table: - raise CriticalError('table \'{}\' has spaces - parsing error for row: {}'.format(table, row)) - if ' ' in database: - raise CriticalError('database \'{}\' has spaces - parsing error for row: {}'.format(database, row)) + if table: + table = table.lower().strip('`') + if ' ' in table: + raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\ + .format(table, self.index_output(row))) + if database: + database = database.lower().strip('`') + if ' ' in database: + raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\ + .format(database, self.index_output(row))) if table == 'null': - raise CriticalError('table == null - parsing error for row: {}'.format(row)) + raise AssertionError('table == null - parsing error for row: {}'.format(row)) return (database, table) def get_db_table_from_resource(self, row): @@ -343,6 +364,8 @@ def output(self, row, database, table): if not self.re_table.match('{}.{}'.format(database, table)): log.warning('%s.%s does not match table regex', database, table) return + # instead of collecting in ram, now just post-process through sort -u + # this way it is easier to see live extractions, --debug and correlate #self.data[database] = self.data.get(database, {}) #self.data[database][table] = 1 if table and not database: @@ -358,18 +381,29 @@ def output(self, row, database, table): def collapse_sql_fields(self, row): sql_index = self.indicies['sql_index'] sql_index2 = self.indicies['sql_index2'] + object_index = self.indicies['object_index'] len_row = len(row) if len_row > self.len_headers: log.debug('collapsing fields in row: %s', row) # divide by 2 to account for this having been done twice in duplicated SQL operational text - difference = (len_row - self.len_headers) / 2 - # slice indicies must be integers - if not isInt(difference): - raise CriticalError("difference in field length '{}' is not an integer for row: {}"\ - .format(difference, row)) - difference = int(difference) - row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) - row = row[:sql_index] + row[sql_index+difference:] + # Update: appears this broke as only 2nd occurence of SQL operational text field got split to new fields, + # which is weird because the log shows both 1st and 2nd SQL text fields were double quoted + difference = len_row - self.len_headers + # seems first occurrence doesn't get split in some occurence, + # wasn't related to open in newline universal mode though + # if 2 fields after isn't the /user/hive/warehouse/blah.db then 1st SQL wasn't split + # would have to regex /user/hive/warehouse/blah.db(?:/table)? + #if not row[sql_index+2].endswith('.db'): + # if object field is TABLE or DATABASE then 1st sql field wasn't split + if row[object_index] not in ('TABLE', 'DATABASE'): + difference /= 2 + # slice indicies must be integers + if not isInt(difference): + raise AssertionError("difference in field length '{}' is not an integer for row: {}"\ + .format(difference, row)) + difference = int(difference) + row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index+difference:] row[sql_index2] = ','.join([self.sql_decomment(_) for _ in row[sql_index2:difference]]) row = row[:sql_index2] + row[sql_index2+difference:] log.debug('collapsed row: %s', row) @@ -381,6 +415,10 @@ def collapse_sql_fields(self, row): def sql_decomment(string): return string.split('--')[0].strip() + @staticmethod + def index_output(obj): + return '\n'.join(['{}\t{}'.format(index, item) for (index, item) in enumerate(obj)]) + if __name__ == '__main__': ClouderaNavigatorTablesUsed().main() From 45cfdfcae980202d45836cf8a94dd3bdeeb53765 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 13:16:35 +0000 Subject: [PATCH 1132/2779] updated travis_last_log.py --- travis_last_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis_last_log.py b/travis_last_log.py index 5005c3417..c703ef300 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -77,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.0' +__version__ = '0.6.1' class TravisLastBuildLog(CLI): @@ -151,7 +151,7 @@ def process_options(self): self.repo = self.get_local_repo_name() if not self.repo: self.usage('--job-id / --repo not specified') - validate_alnum(self.travis_token, 'travis token') + validate_alnum(self.travis_token, 'travis token', is_secret=True) self.headers['Authorization'] = 'token {0}'.format(self.travis_token) self.num = self.get_opt('num') validate_int(self.num, 'num', 1) From 281b32e1eb6ecadd1d75d12cb7db8fc87e44fe65 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 13:16:45 +0000 Subject: [PATCH 1133/2779] updated travis_debug_session.py --- travis_debug_session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis_debug_session.py b/travis_debug_session.py index eb2557d7d..805fecdd0 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -69,7 +69,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.0' +__version__ = '0.9.1' class TravisDebugSession(CLI): @@ -155,7 +155,7 @@ def process_options(self): self.repo = self.get_local_repo_name() if not self.repo: self.usage('--job-id / --repo not specified') - validate_alnum(self.travis_token, 'travis token') + validate_alnum(self.travis_token, 'travis token', is_secret=True) self.headers['Authorization'] = 'token {0}'.format(self.travis_token) @staticmethod From 436cf262a6862cce124d820c67c09daba350c2ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 13:17:38 +0000 Subject: [PATCH 1134/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7b9ecd187..bcf1929dd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7b9ecd187bf79e02f996915c4a606bbfa9da19e8 +Subproject commit bcf1929ddb1bf016b6c29d02e7482eb003735c74 From eeb390d1f6b20e1fe0edd879d04c1b3c0a7f4a81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 13:17:39 +0000 Subject: [PATCH 1135/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fe836adba..13d728ab8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fe836adba0c49c3a0b3316318653200469dcb6e1 +Subproject commit 13d728ab831def658188c07d4130074ef0cfd214 From 94caafbc5ac3b9871c519b1c436b92a48c1bb29f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 14:17:23 +0000 Subject: [PATCH 1136/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 43 +++++++++++++++++++------------ 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index eb1cee773..57902b25e 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -52,6 +52,7 @@ #from __future__ import unicode_literals import csv +import gzip import logging import os import re @@ -72,7 +73,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' class ClouderaNavigatorTablesUsed(CLI): @@ -85,20 +86,10 @@ def __init__(self): self.delimiter = None self.quotechar = None self.escapechar = None + self.timeout_default = None #self.data = {} self.indicies = {} self.len_headers = None - self.operations_to_ignore = [ - '', - 'HIVEREPLICATIONCOMMAND', - 'START', - 'STOP', - 'RESTART', - 'LOAD', - 'SWITCHDATABASE', - 'USE', - ] - self.timeout_default = None self.table_regex = r'[\w\.`]+' self.re_table = re.compile(self.table_regex) # doesn't handle JOINs because SQL pros usually use table aliases @@ -111,6 +102,16 @@ def __init__(self): self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') self.re_ignored_users = None self.csv_writer = None + self.operations_to_ignore = [ + '', + 'HIVEREPLICATIONCOMMAND', + 'START', + 'STOP', + 'RESTART', + 'LOAD', + 'SWITCHDATABASE', + 'USE', + ] def add_options(self): super(ClouderaNavigatorTablesUsed, self).add_options() @@ -152,8 +153,12 @@ def run(self): quoting=quoting, fieldnames=fieldnames) for filename in self.args: - with open(filename, 'rU') as filehandle: - self.process_file(filehandle) + if filename.endswith('.gz'): + with gzip.open(filename, 'rU') as filehandle: + self.process_file(filehandle) + else: + with open(filename, 'rU') as filehandle: + self.process_file(filehandle) #csv_writer.writeheader() #for database in sorted(self.data): @@ -267,8 +272,7 @@ def process_rows(self, csv_reader): #log.debug('current row = %s', current_row) if not current_row: continue - # originally did this by counting fields but SQL fragmentation generates extra fields - if self.re_timestamp.match(current_row[0]): + if self.is_new_record(current_row): row = last_row last_row = current_row else: @@ -279,6 +283,10 @@ def process_rows(self, csv_reader): self.process_row(row) self.process_row(last_row) + # originally did this by counting fields but SQL fragmentation generates extra fields + def is_new_record(self, current_row): + return self.re_timestamp.match(current_row[0]) + def process_row(self, row): if not row: return @@ -297,7 +305,8 @@ def process_row(self, row): def parse_table(self, row): #log.debug(row) user = row[self.indicies['user_index']] - # 'hari.sekhon' in 'hari.sekhon@somedomain.com' in kerberos + # user: 'hari.sekhon' + # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com' if self.re_ignored_users and self.re_ignored_users.match(user): log.debug('skipping row for ignored user %s: %s', user, row) return (None, None) From ae413f806d031b092d6e6448d05f6a069c7040f2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 14:27:10 +0000 Subject: [PATCH 1137/2779] added cloudera_navigator_tables_used_postgres.py --- cloudera_navigator_tables_used_postgres.py | 218 +++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100755 cloudera_navigator_tables_used_postgres.py diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py new file mode 100755 index 000000000..7de004292 --- /dev/null +++ b/cloudera_navigator_tables_used_postgres.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-16 19:21:24 +0000 (Mon, 16 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Processes Cloudera Navigator CSV logs exported from PostgreSQL to list the tables used (SELECT'ed from) + +This allows you to see if you're wasting time maintaining datasets nobody is using + +See cloudera_navigator_audit_logs_export_postgres.sh for a script to export these logs + +Supports reading directly from gzipped logs if they end in .gz file extension + +./cloudera_navigator_tables_used_postgres.py nav.public.hive_audit_events_2019_11_*.csv.gz \\ + nav.public.impala_audit_events_2019_11_*.csv.gz ... + +Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison): + +"database","table" + +Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import csv +#import logging +import os +import re +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +import gzip +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, isInt + #from harisekhon import CLI + from cloudera_navigator_tables_used import ClouderaNavigatorTablesUsed +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class ClouderaNavigatorTablesUsedPostgreSQL(ClouderaNavigatorTablesUsed): + + def __init__(self): + # Python 2.x + super(ClouderaNavigatorTablesUsedPostgreSQL, self).__init__() + # Python 3.x + # super().__init__() + # recombine records due to SQL \n breaking up records, new records start like: + # 306529,1574163624392,1,hive, + self.re_new_record = re.compile(r'^\d+,\d+,[01],(?:hive|impala),') + # get db + table from resource path (just one layer of checks) + self.re_resource = re.compile(r'/(\w+)\.db/(\w+)') + +# Navigator table logs: +# +# gzcat nav.public.hive_audit_events_2019_11_19.csv.gz | csv_header_indices.sh +# 0 id +# 1 event_time +# 2 allowed +# 3 service_name +# 4 username +# 5 ip_addr +# 6 operation +# 7 database_name +# 8 object_type +# 9 table_name +# 10 operation_text +# 11 impersonator +# 12 resource_path +# 13 object_usage_type + + def process_file(self, filehandle): + csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') + headers = csv_reader.next() + self.len_headers = len(headers) + # needed to ensure row joining works later on with number of fields left + assert self.len_headers == 14 + user_index = 4 + operation_index = 6 + database_index = 7 + table_index = 9 + sql_index = 10 + resource_index = 12 + assert headers[user_index] == 'username' + assert headers[operation_index] == 'operation' + assert headers[table_index] == 'table_name' + assert headers[database_index] == 'database_name' + assert headers[sql_index] == 'operation_text' + assert headers[resource_index] == 'resource_path' + self.indicies = { + 'user_index': user_index, + 'operation_index': operation_index, + 'resource_index': resource_index, + 'table_index': table_index, + 'database_index': database_index, + 'sql_index': sql_index, + } + self.process_rows(csv_reader) + + def is_new_record(self, current_row): + return self.re_new_record.match(','.join(current_row)) + + def parse_table(self, row): + #log.debug(row) + user = row[self.indicies['user_index']] + # user: 'hari.sekhon' + # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com' + if self.re_ignored_users and self.re_ignored_users.match(user): + log.debug('skipping row for ignored user %s: %s', user, row) + return (None, None) + database = row[self.indicies['database_index']].strip() + table = row[self.indicies['table_index']].strip() + if not database or not table or not self.re_table.match('{}.{}'.format(database, table)): + #log.info('table not found in fields for row: %s', row) + operation = row[self.indicies['operation_index']] + if operation in self.operations_to_ignore: + return (None, None) + elif operation == 'QUERY': + query = row[self.indicies['sql_index']] + # cheaper than re_ignore to pre-filter + if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'): + return (None, None) + (database, table) = self.get_db_table_from_resource(row) + if database and table: + pass + else: + log.debug('database/table not found in row: %s', row) + log.debug('trying to parse: %s', query) + match = self.re_select_from_table.search(query) + if match: + table = match.group(1) + if '.' in table: + (database, table) = table.split('.', 1) + # could use .search but all these seem to be at beginning + elif self.re_ignore.match(query): + return (None, None) + else: + log.warning('failed to parse database/table from query: %s', query) + return (None, None) + else: + log.debug('database/table not found in row and operation is not a query to parse: %s', row) + return (None, None) + if not table and not database: + return (None, None) + if table: + table = table.lower().strip('`') + if ' ' in table: + raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\ + .format(table, self.index_output(row))) + if database: + database = database.lower().strip('`') + if ' ' in database: + raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\ + .format(database, self.index_output(row))) + if table == 'null': + raise AssertionError('table == null - parsing error for row: {}'.format(row)) + return (database, table) + + def get_db_table_from_resource(self, row): + database = None + table = None + resource = row[self.indicies['resource_index']] + if resource: + match = self.re_resource.search(resource) + if match: + database = match.group(1) + table = match.group(2) + return (database, table) + + def collapse_sql_fields(self, row): + sql_index = self.indicies['sql_index'] + len_row = len(row) + if len_row > self.len_headers: + log.debug('collapsing fields in row: %s', row) + difference = len_row - self.len_headers + # slice indicies must be integers + if not isInt(difference): + raise AssertionError("difference in field length '{}' is not an integer for row: {}"\ + .format(difference, row)) + difference = int(difference) + row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index+difference:] + log.debug('collapsed row: %s', row) + else: + log.debug('not collapsing row: %s', row) + return row + + +if __name__ == '__main__': + ClouderaNavigatorTablesUsedPostgreSQL().main() From a3f598c5373e3e6c848127b2450d2f481f7bbbd1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 14:43:12 +0000 Subject: [PATCH 1138/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 57902b25e..616e6de72 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -96,7 +96,17 @@ def __init__(self): self.re_select_from_table = re.compile(r'\bSELECT\b.+\bFROM\b(?:\s|\n)+({table_regex})'\ .format(table_regex=self.table_regex), \ re.I | re.MULTILINE | re.DOTALL) - self.re_ignore = re.compile(r'\b(?:SHOW|DESCRIBE|USE|REFRESH|INVALIDATE\S+METADATA|GET_TABLES|GET_SCHEMAS)\b',\ + ignore_statements = [ + 'SHOW', + 'DESCRIBE', + 'USE', + 'REFRESH', + r'INVALIDATE\S+METADATA', + 'GET_TABLES', + 'GET_SCHEMAS', + 'VIEW_METADATA', + ] + self.re_ignore = re.compile(r'\b(?:' + '|'.join(ignore_statements) + ')\b',\ re.I | re.MULTILINE | re.DOTALL) # 2020-01-31T20:45:59.000Z self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') From e01917971d5888ce89cd01d6a3b14d57fdc53e9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 14:54:34 +0000 Subject: [PATCH 1139/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 616e6de72..01297e695 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -32,6 +32,9 @@ - matches user or kerberos principal - eg. to omit ETL service account from skewing data access results +Supports reading directly from gzipped logs if they end in .gz file extension. +However, the gzip library may have issues around universal newline support, if so, gunzip first. + See cloudera_navigator_audit_logs_download.sh for a script to export these logs ./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \\ @@ -164,9 +167,11 @@ def run(self): fieldnames=fieldnames) for filename in self.args: if filename.endswith('.gz'): + log.debug("processing gzip'd file: %s", filename) with gzip.open(filename, 'rU') as filehandle: self.process_file(filehandle) else: + log.debug("processing file: %s", filename) with open(filename, 'rU') as filehandle: self.process_file(filehandle) From 7c47723e338798083072e7c445add7d57f42a67f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:12:19 +0000 Subject: [PATCH 1140/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 01297e695..f479ca06f 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -67,7 +67,7 @@ sys.path.append(lib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log, validate_regex, isInt + from harisekhon.utils import log, validate_regex, isInt, isPythonMinVersion from harisekhon import CLI except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) @@ -86,6 +86,7 @@ def __init__(self): super(ClouderaNavigatorTablesUsed, self).__init__() # Python 3.x # super().__init__() + csv.field_size_limit(sys.maxsize) self.delimiter = None self.quotechar = None self.escapechar = None @@ -109,7 +110,7 @@ def __init__(self): 'GET_SCHEMAS', 'VIEW_METADATA', ] - self.re_ignore = re.compile(r'\b(?:' + '|'.join(ignore_statements) + ')\b',\ + self.re_ignore = re.compile(r'\b(?:' + '|'.join(ignore_statements) + r')\b',\ re.I | re.MULTILINE | re.DOTALL) # 2020-01-31T20:45:59.000Z self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') @@ -165,14 +166,17 @@ def run(self): escapechar=self.escapechar, quoting=quoting, fieldnames=fieldnames) + mode = 'rtU' + if isPythonMinVersion(3): + mode = 'rt' for filename in self.args: if filename.endswith('.gz'): log.debug("processing gzip'd file: %s", filename) - with gzip.open(filename, 'rU') as filehandle: + with gzip.open(filename, mode, encoding="utf8") as filehandle: self.process_file(filehandle) else: log.debug("processing file: %s", filename) - with open(filename, 'rU') as filehandle: + with open(filename, mode, encoding="utf8") as filehandle: self.process_file(filehandle) #csv_writer.writeheader() @@ -237,7 +241,12 @@ def run(self): def process_file(self, filehandle): csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') - headers = csv_reader.next() + try: + # Python 2 + headers = csv_reader.next() + except AttributeError: + # Python 3 + headers = next(csv_reader) self.len_headers = len(headers) # needed to ensure row joining works later on with number of fields left assert self.len_headers == 44 From a84227d8b72030a87e4369c87e3cd64b445be533 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:18:16 +0000 Subject: [PATCH 1141/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index f479ca06f..f1c66c350 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -169,14 +169,15 @@ def run(self): mode = 'rtU' if isPythonMinVersion(3): mode = 'rt' + # open(..., encoding="utf-8") is Python 3 only - uses system default otherwise for filename in self.args: if filename.endswith('.gz'): log.debug("processing gzip'd file: %s", filename) - with gzip.open(filename, mode, encoding="utf8") as filehandle: + with gzip.open(filename, mode) as filehandle: self.process_file(filehandle) else: log.debug("processing file: %s", filename) - with open(filename, mode, encoding="utf8") as filehandle: + with open(filename, mode) as filehandle: self.process_file(filehandle) #csv_writer.writeheader() From be910fc932ead11113061e4169f94b9d2e3172b9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:36:01 +0000 Subject: [PATCH 1142/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index f1c66c350..6f73105b2 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -104,13 +104,20 @@ def __init__(self): 'SHOW', 'DESCRIBE', 'USE', - 'REFRESH', - r'INVALIDATE\S+METADATA', + 'CREATE', + 'DROP', + 'INSERT', + 'DELETE', + 'UPDATE', 'GET_TABLES', 'GET_SCHEMAS', 'VIEW_METADATA', + 'ANALYZE', + r'COMPUTE\s+STATS' + 'REFRESH', + r'INVALIDATE\s+METADATA', ] - self.re_ignore = re.compile(r'\b(?:' + '|'.join(ignore_statements) + r')\b',\ + self.re_ignore = re.compile(r'^\s*\b(?:' + '|'.join(ignore_statements) + r')\b',\ re.I | re.MULTILINE | re.DOTALL) # 2020-01-31T20:45:59.000Z self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') From b06c6f1ba6a7557aa2ac2239b52ca4fcf88a2908 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:40:14 +0000 Subject: [PATCH 1143/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bcf1929dd..621e7e2fd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bcf1929ddb1bf016b6c29d02e7482eb003735c74 +Subproject commit 621e7e2fdf660122d94549a5630e518fee66cf3d From 02fe7a948abe24fce5be4c04a630cb8e43f8f2d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 15:41:07 +0000 Subject: [PATCH 1144/2779] updated cloudera_navigator_tables_used_postgres.py --- cloudera_navigator_tables_used_postgres.py | 69 ++++++++++++++++------ 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py index 7de004292..7c994b529 100755 --- a/cloudera_navigator_tables_used_postgres.py +++ b/cloudera_navigator_tables_used_postgres.py @@ -22,10 +22,11 @@ See cloudera_navigator_audit_logs_export_postgres.sh for a script to export these logs -Supports reading directly from gzipped logs if they end in .gz file extension +Supports reading directly from gzipped logs if they end in .gz file extension. +However, the gzip library may have issues around universal newline support, if so, gunzip first. -./cloudera_navigator_tables_used_postgres.py nav.public.hive_audit_events_2019_11_*.csv.gz \\ - nav.public.impala_audit_events_2019_11_*.csv.gz ... +./cloudera_navigator_tables_used_postgres.py nav.public.hive_audit_events_*.csv.gz \\ + nav.public.impala_audit_events_*.csv.gz ... Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison): @@ -63,7 +64,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.2.0' class ClouderaNavigatorTablesUsedPostgreSQL(ClouderaNavigatorTablesUsed): @@ -97,24 +98,55 @@ def __init__(self): # 12 resource_path # 13 object_usage_type +# gzcat nav.public.impala_audit_events_2019_11_19.csv.gz | csv_header_indices.sh +# 0 id +# 1 event_time +# 2 allowed +# 3 service_name +# 4 username +# 5 impersonator +# 6 ip_addr +# 7 operation +# 8 query_id +# 9 session_id +# 10 status +# 11 database_name +# 12 object_type +# 13 table_name +# 14 privilege +# 15 operation_text + def process_file(self, filehandle): csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') headers = csv_reader.next() self.len_headers = len(headers) # needed to ensure row joining works later on with number of fields left - assert self.len_headers == 14 - user_index = 4 - operation_index = 6 - database_index = 7 - table_index = 9 - sql_index = 10 - resource_index = 12 - assert headers[user_index] == 'username' - assert headers[operation_index] == 'operation' - assert headers[table_index] == 'table_name' - assert headers[database_index] == 'database_name' - assert headers[sql_index] == 'operation_text' - assert headers[resource_index] == 'resource_path' + assert self.len_headers == 14 or self.len_headers == 16 + # Hive postgres audit log + if self.len_headers == 14: + user_index = 4 + operation_index = 6 + database_index = 7 + table_index = 9 + sql_index = 10 + resource_index = 12 + assert headers[user_index] == 'username' + assert headers[resource_index] == 'resource_path' + # Impala postgres audit log + elif self.len_headers == 16: + user_index = 5 # impersonator field contains actual user, user field is always 'impala' + operation_index = 7 + database_index = 11 + table_index = 13 + sql_index = 15 + resource_index = None + assert headers[user_index] == 'impersonator' + else: + raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala') + assert headers[sql_index] == 'operation_text' + assert headers[database_index] == 'database_name' + assert headers[table_index] == 'table_name' + assert headers[operation_index] == 'operation' self.indicies = { 'user_index': user_index, 'operation_index': operation_index, @@ -185,6 +217,9 @@ def parse_table(self, row): return (database, table) def get_db_table_from_resource(self, row): + # only available for hive audit logs, not impala + if self.indicies['resource_index'] is None: + return (None, None) database = None table = None resource = row[self.indicies['resource_index']] From 091495164b13dbfadced36ee2913ea798db73e45 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 16:24:48 +0000 Subject: [PATCH 1145/2779] updated aws_users_last_used.py --- aws_users_last_used.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 588a1d7af..3c467292b 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -67,7 +67,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' class AWSUsersLastUsed(CLI): @@ -112,7 +112,7 @@ def run(self): filehandle = StringIO(unicode(csv_content)) filehandle.seek(0) csvreader = csv.reader(filehandle) - headers = csvreader.next() + headers = next(csvreader) assert headers[0] == 'user' assert headers[4] == 'password_last_used' assert headers[10] == 'access_key_1_last_used_date' From cb28e00d175c5052ee4c0a19e61b5baa0a41e10d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 16:24:54 +0000 Subject: [PATCH 1146/2779] updated aws_users_unused_access_keys.py --- aws_users_unused_access_keys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py index 9b9083eb9..fcb244e07 100755 --- a/aws_users_unused_access_keys.py +++ b/aws_users_unused_access_keys.py @@ -65,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.2.1' class AWSUnusedAccessKeys(CLI): @@ -113,7 +113,7 @@ def run(self): filehandle = StringIO(unicode(csv_content)) filehandle.seek(0) csvreader = csv.reader(filehandle) - headers = csvreader.next() + headers = next(csvreader) log.debug('headers: %s', headers) assert headers[0] == 'user' assert headers[8] == 'access_key_1_active' From eda0798c7920ee8d923e6c4a77d2b22f0a820aed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 17:34:27 +0000 Subject: [PATCH 1147/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 6f73105b2..cbd95d6da 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -166,6 +166,7 @@ def run(self): if self.quotechar == '': quoting = csv.QUOTE_NONE + #fieldnames = ['database', 'table', 'user'] fieldnames = ['database', 'table'] self.csv_writer = csv.DictWriter(sys.stdout, delimiter=self.delimiter, @@ -415,6 +416,7 @@ def output(self, row, database, table): log.info('got database but not table for row: %s', row) if not table and not database: return + #self.csv_writer.writerow({'database': database, 'table': table, 'user': row[self.indicies['user_index']]}) self.csv_writer.writerow({'database': database, 'table': table}) if log.isEnabledFor(logging.DEBUG): sys.stdout.flush() From 6db6bf7e60cd4b7173feed8b283d3b7e29b0cd44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 17:34:35 +0000 Subject: [PATCH 1148/2779] updated cloudera_navigator_tables_used_postgres.py --- cloudera_navigator_tables_used_postgres.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py index 7c994b529..7d2fcaee9 100755 --- a/cloudera_navigator_tables_used_postgres.py +++ b/cloudera_navigator_tables_used_postgres.py @@ -122,25 +122,23 @@ def process_file(self, filehandle): self.len_headers = len(headers) # needed to ensure row joining works later on with number of fields left assert self.len_headers == 14 or self.len_headers == 16 + user_index = 4 + assert headers[user_index] == 'username' # Hive postgres audit log if self.len_headers == 14: - user_index = 4 operation_index = 6 database_index = 7 table_index = 9 sql_index = 10 resource_index = 12 - assert headers[user_index] == 'username' assert headers[resource_index] == 'resource_path' # Impala postgres audit log elif self.len_headers == 16: - user_index = 5 # impersonator field contains actual user, user field is always 'impala' operation_index = 7 database_index = 11 table_index = 13 sql_index = 15 resource_index = None - assert headers[user_index] == 'impersonator' else: raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala') assert headers[sql_index] == 'operation_text' From 8fa7e4c15e7d469b8356939c0d7a736d17851afb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Mar 2020 17:37:40 +0000 Subject: [PATCH 1149/2779] updated cloudera_navigator_tables_used.py --- cloudera_navigator_tables_used.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index cbd95d6da..645d344b4 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -113,7 +113,7 @@ def __init__(self): 'GET_SCHEMAS', 'VIEW_METADATA', 'ANALYZE', - r'COMPUTE\s+STATS' + r'COMPUTE\s+STATS', 'REFRESH', r'INVALIDATE\s+METADATA', ] From a6391e6e34b6030ba8d9a6fb5e085519f76e288c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Mar 2020 13:45:50 +0000 Subject: [PATCH 1150/2779] added .concourse.yml --- .concourse.yml | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .concourse.yml diff --git a/.concourse.yml b/.concourse.yml new file mode 100644 index 000000000..d50f7ecf4 --- /dev/null +++ b/.concourse.yml @@ -0,0 +1,59 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-21 11:06:48 +0000 (Sat, 21 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +resources: +- name: github + icon: github-circle + type: git + source: + uri: https://github.com/harisekhon/devops-python-tools + branch: master +#- name: daily +# type: time +# source: +# interval: 1d + +# https://concourse-ci.org/golang-library-example.html + +jobs: +- name: build + public: false + plan: + - get: github # from resource above + trigger: true + #version: every # build every git commit, default: latest + - task: build + config: + platform: linux + image_resource: + type: docker-image + source: + repository: ubuntu + tag: latest + inputs: + - name: github + path: code + params: + CONCOURSE: 1 + run: + path: /bin/bash + args: + - -c + - | + cd code && + apt update -q && + apt install -qy git make && + make init && + make ci test From e84131dd049aeec5618db68da2ca4d6fcb3337e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:15:36 +0000 Subject: [PATCH 1151/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 680feb726..5770ebfe3 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Hari Sekhon - DevOps Python Tools [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) +[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 9594dcbd63893015af7dc2438e2b0c037e752600 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:20:59 +0000 Subject: [PATCH 1152/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5770ebfe3..957b93c43 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Hari Sekhon - DevOps Python Tools [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools)](https://github.com/harisekhon/devops-python-tools/network) From cfbb467e9de520257831ea9494f25f334485b37a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:28:15 +0000 Subject: [PATCH 1153/2779] updated aws_users_last_used.py --- aws_users_last_used.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 3c467292b..9a93b7bad 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -54,7 +54,6 @@ from io import StringIO from math import floor import boto3 -from botocore.exceptions import ClientError srcdir = os.path.abspath(os.path.dirname(__file__)) libdir = os.path.join(srcdir, 'pylib') sys.path.append(libdir) From d8821908b574b8637799eca69de74559d3d511f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:34:21 +0000 Subject: [PATCH 1154/2779] updated dockerhub_show_tags.py --- dockerhub_show_tags.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dockerhub_show_tags.py b/dockerhub_show_tags.py index 88c9ddc08..e0dfecbb8 100755 --- a/dockerhub_show_tags.py +++ b/dockerhub_show_tags.py @@ -32,6 +32,7 @@ import json import logging import os +import re import sys import traceback import urllib @@ -52,7 +53,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.2' +__version__ = '0.6.3' class DockerHubTags(CLI): @@ -79,8 +80,11 @@ def run(self): self.usage('no repos given as args') self.quiet = self.get_opt('quiet') if not self.quiet: + # cheaper but lgtm hassling me, not a security issue but will shut them up print('\nDocker', end='') - if 'registry.hub.docker.com' in self.url_base: + #if 'registry.hub.docker.com' in self.url_base: + # match anchors but I prefer explicit anchor, more intuitive for other generic language coders + if re.match(r'^https://registry\.hub\.docker\.com/', self.url_base): print('Hub') else: print(' Registry: {0}'.format(self.url_base.split('/v2', 1)[0])) From 208b3d82cf74ae30bebe01c5ed391776ec7f632d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:37:44 +0000 Subject: [PATCH 1155/2779] updated dockerfiles_check_git_branches.py --- dockerfiles_check_git_branches.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dockerfiles_check_git_branches.py b/dockerfiles_check_git_branches.py index a7edf5195..b2e407f4b 100755 --- a/dockerfiles_check_git_branches.py +++ b/dockerfiles_check_git_branches.py @@ -91,7 +91,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.2' +__version__ = '0.7.3' class DockerfileGitBranchCheckTool(CLI): @@ -321,8 +321,8 @@ def check_file(self, filename, branch): def check_dockerfile_arg(self, filename, branch): log.debug('check_dockerfile_arg({0}, {1})'.format(filename, branch)) - branch_base = str(branch).replace('-dev', '') - (branch_base, branch_versions) = self.branch_version(branch) + branch_stripped = str(branch).replace('-dev', '') + (branch_base, branch_versions) = self.branch_version(branch_stripped) with open(filename) as filehandle: version_index = 0 for line in filehandle: From 62215f42f321b2a6c622ba241dbe3775b5417f1c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:38:31 +0000 Subject: [PATCH 1156/2779] updated anonymize.py --- anonymize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonymize.py b/anonymize.py index 8e84d0c5f..418427711 100755 --- a/anonymize.py +++ b/anonymize.py @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.11' +__version__ = '0.10.12' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -219,7 +219,7 @@ def __init__(self): #'countryCode', 'displayName', 'displayNamePrintable', - 'division' + 'division', 'employeeID', 'groupMembershipSAM', 'info', From 2e6c324f451ff2e483db95cd18ea7ac2c9ca89ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:49:22 +0000 Subject: [PATCH 1157/2779] refactored out generate sql static method for overriding --- hive_tables_null_rows.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 7ab8b53dd..82082d358 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -62,7 +62,7 @@ __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.6.0' class HiveTablesNullRows(HiveForEachTable): @@ -79,11 +79,11 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, _query): + def execute(self, conn, database, table, _): columns = [] log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec + # impala library doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) #column_cursor.execute('describe %(table)s', {'table': table}) column_cursor.execute('use `{}`'.format(database)) @@ -92,9 +92,7 @@ def execute(self, conn, database, table, _query): column = column_row[0] #column_type = column_row[1] columns.append(column) - query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ - .format(db=database, table=table) + \ - "` IS NULL AND `".join(columns) + "` IS NULL" + query = self.generate_sql(database, table, columns) with conn.cursor() as table_cursor: log.debug('executing query: %s', query) table_cursor.execute(query) @@ -102,6 +100,13 @@ def execute(self, conn, database, table, _query): count = result[0] print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) + @staticmethod + def generate_sql(database, table, columns): + sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL AND `".join(columns) + "` IS NULL" + return sql + if __name__ == '__main__': HiveTablesNullRows().main() From f08bb8cdca5db6869cf28c682e86a3f27eeadf21 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:50:19 +0000 Subject: [PATCH 1158/2779] deduped code using inheriting overriding of hive_tables_null_rows --- hive_tables_row_counts_any_nulls.py | 54 +++++++---------------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index ce17ca99a..a1de5e461 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -52,8 +52,7 @@ sys.path.append(pylib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log - from hive_foreach_table import HiveForEachTable + from hive_tables_null_rows import HiveTablesNullRows except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) print("Did you remember to build the project by running 'make'?", file=sys.stderr) @@ -62,45 +61,18 @@ __author__ = 'Hari Sekhon' -__version__ = '0.5.0' - - -class HiveTablesRowsWithNulls(HiveForEachTable): - - def __init__(self): - # Python 2.x - super(HiveTablesRowsWithNulls, self).__init__() - # Python 3.x - # super().__init__() - self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option - self.database = None - self.table = None - #self.partition = None - self.ignore_errors = False - - # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, _query): - columns = [] - log.info("describing table '%s.%s'", database, table) - with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec - #column_cursor.execute('use %(database)s', {'database': database}) - #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use `{}`'.format(database)) - column_cursor.execute('describe `{}`'.format(table)) - for column_row in column_cursor: - column = column_row[0] - #column_type = column_row[1] - columns.append(column) - query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ - .format(db=database, table=table) + \ - "` IS NULL OR `".join(columns) + "` IS NULL" - with conn.cursor() as table_cursor: - log.debug('executing query: %s', query) - table_cursor.execute(query) - for result in table_cursor: - count = result[0] - print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) +__version__ = '0.6.0' + + +class HiveTablesRowsWithNulls(HiveTablesNullRows): + + @staticmethod + def generate_sql(database, table, columns): + # impala library doesn't support parameterized query quoting from dbapi spec + sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL OR `".join(columns) + "` IS NULL" + return sql if __name__ == '__main__': From 8fb1e49f80a32690a98741d04b7fc477ade72873 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:54:05 +0000 Subject: [PATCH 1159/2779] updated anonymize.py --- anonymize.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/anonymize.py b/anonymize.py index 418427711..d46b6c796 100755 --- a/anonymize.py +++ b/anonymize.py @@ -72,18 +72,20 @@ # pylint: disable=unused-import from harisekhon.utils import \ aws_host_ip_regex, \ - domain_regex, \ domain_regex_strict, \ - email_regex, \ filename_regex, \ fqdn_regex, \ host_regex, \ hostname_regex, \ ip_prefix_regex, \ ip_regex, \ - mac_regex, \ subnet_mask_regex, \ user_regex + from harisekhon.utils import \ + domain_regex, \ + email_regex, \ + mac_regex \ + # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend from harisekhon import CLI except ImportError as _: print(traceback.format_exc(), end='') From cb43cbb372912c4c11cf6a18c21b8d52511855af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 12:59:25 +0000 Subject: [PATCH 1160/2779] updated validate_avro.py --- validate_avro.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_avro.py b/validate_avro.py index f8a896466..f81c3afb7 100755 --- a/validate_avro.py +++ b/validate_avro.py @@ -39,7 +39,7 @@ try: from avro3.datafile import DataFileReader, DataFileException from avro3.io import DatumReader -except: +except Exception: # pylint: disable=broad-except from avro.datafile import DataFileReader, DataFileException from avro.io import DatumReader libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) @@ -55,7 +55,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.1' +__version__ = '0.9.2' class AvroValidatorTool(CLI): From f1f2ee3f4bf2aa93ef58b07c7d38bb0527503f6f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 13:00:29 +0000 Subject: [PATCH 1161/2779] updated hbase_flush_tables.py --- hbase_flush_tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase_flush_tables.py b/hbase_flush_tables.py index c4f43ef8d..04d51bc75 100755 --- a/hbase_flush_tables.py +++ b/hbase_flush_tables.py @@ -44,7 +44,7 @@ import sys import traceback import subprocess -from subprocess import PIPE +PIPE = subprocess.PIPE libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) try: From 71df727098d340a786154f2435f51add299a27e7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 13:02:54 +0000 Subject: [PATCH 1162/2779] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index 955b395e7..efcbc1279 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -60,11 +60,11 @@ from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error -from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import +from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error __author__ = 'Hari Sekhon' -__version__ = '0.8.0' +__version__ = '0.8.1' class SparkCSVToAvro(CLI): From 20e0d0c4d6b73840405a2388d6ec65a2839651f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 13:03:15 +0000 Subject: [PATCH 1163/2779] updated spark_csv_to_parquet.py --- spark_csv_to_parquet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py index f3f6909e6..3bbb9c18a 100755 --- a/spark_csv_to_parquet.py +++ b/spark_csv_to_parquet.py @@ -52,11 +52,11 @@ from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error -from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import +from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error __author__ = 'Hari Sekhon' -__version__ = '0.8.0' +__version__ = '0.8.1' class SparkCSVToParquet(CLI): From a541399757edfd051a50eda9be2703d1b82a960b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 25 Mar 2020 13:03:46 +0000 Subject: [PATCH 1164/2779] updated validate_multimedia.py --- validate_multimedia.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validate_multimedia.py b/validate_multimedia.py index 9452fd6c7..54def6deb 100755 --- a/validate_multimedia.py +++ b/validate_multimedia.py @@ -45,7 +45,7 @@ import re import sys import subprocess -from subprocess import CalledProcessError +CalledProcessError = subprocess.CalledProcessError libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) try: @@ -164,8 +164,8 @@ def check_path(self, path): die("failed to determine if path '%s' is file or directory" % path) def check_media_file(self, filename): - if self.is_excluded(filename): - return + #if self.is_excluded(filename): + # return valid_media_msg = '%s => OK' % filename invalid_media_msg = '%s => INVALID' % filename cmd = self.validate_cmd From 071afdd59ed494b7fdd1990dc28fcbfb6b550f6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:49:58 +0000 Subject: [PATCH 1165/2779] added buddy.yml --- buddy.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 buddy.yml diff --git a/buddy.yml b/buddy.yml new file mode 100644 index 000000000..9b897a700 --- /dev/null +++ b/buddy.yml @@ -0,0 +1,38 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://buddy.works/docs/yaml/yaml-schema + +- pipeline: "Build" + trigger_mode: "ON_EVERY_PUSH" + ref_name: "master" + ref_type: "BRANCH" + target_site_url: "https://github.com/harisekhon/devops-python-tools" + trigger_condition: "ALWAYS" + actions: + - action: "Execute: make ci test" + type: "BUILD" + working_directory: "/buddy/devops-python-tools" + docker_image_name: "library/ubuntu" + docker_image_tag: "18.04" + execute_commands: + - "apt update &&" + - "apt install -qy make &&" + - "make init &&" + - "make ci test" + volume_mappings: + - "/:/buddy/devops-python-tools" + shell: "BASH" + trigger_condition: "ALWAYS" From b764c9f6cd49011023128ab3884fcba8f9a83efd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:50:05 +0000 Subject: [PATCH 1166/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 621e7e2fd..da52b0d76 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 621e7e2fdf660122d94549a5630e518fee66cf3d +Subproject commit da52b0d76dc4d0db3189f668ea3b01ccc735ac3c From f692ec345f1c53894ec5bb67d573df0501e9b18c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:50:05 +0000 Subject: [PATCH 1167/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 13d728ab8..c657aea1e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 13d728ab831def658188c07d4130074ef0cfd214 +Subproject commit c657aea1e99f9f811faa9aa4980b44153664e0c3 From 0bd5cff97e6b56d4259bbe7f26d9875e3d9637b0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:50:28 +0000 Subject: [PATCH 1168/2779] updated README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 957b93c43..4b1e73070 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,10 @@ Hari Sekhon - DevOps Python Tools [![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/lib/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) -[![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) +[![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) +[![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.concourse.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 9b1bf6d09a89cdd47e98c1c41b033d629f91596f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 01:54:15 +0000 Subject: [PATCH 1169/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b1e73070..1b1d18109 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Hari Sekhon - DevOps Python Tools [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) -[![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.concourse.yml) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From db6782f56c4e2236f14d56d485f1e4bdd06ddb1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 02:01:14 +0000 Subject: [PATCH 1170/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b1d18109..cc3c36f59 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Hari Sekhon - DevOps Python Tools [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/lib/dashboard/jobs) +[![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) From 86dfaa8a6f4af4a57e1fb5e4f2ea908cc4c41d8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 02:02:16 +0000 Subject: [PATCH 1171/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cc3c36f59..3afe9935f 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Hari Sekhon - DevOps Python Tools [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://img.shields.io/shippable/5e52c63645c70f0007ff5152/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) +[![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) From f23adf59b72d97f5a78a237096ab14b0a1245beb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 11:39:42 +0000 Subject: [PATCH 1172/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index da52b0d76..16b436757 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit da52b0d76dc4d0db3189f668ea3b01ccc735ac3c +Subproject commit 16b4367579ff71056b3bdfb62b19550bf334c193 From 04c5217e1a298755a0e8aab7d89149b74bf7b4dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 26 Mar 2020 11:39:42 +0000 Subject: [PATCH 1173/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c657aea1e..cd591772c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c657aea1e99f9f811faa9aa4980b44153664e0c3 +Subproject commit cd591772cb701831e207fa253aef56920ca7fc8d From 643c1c31819d98186804501f061beb49614cf39f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:08:32 +0000 Subject: [PATCH 1174/2779] added gocd_config_repo.json --- setup/gocd_config_repo.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 setup/gocd_config_repo.json diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json new file mode 100644 index 000000000..a3a8beb73 --- /dev/null +++ b/setup/gocd_config_repo.json @@ -0,0 +1,26 @@ +{ + "id": "devops-python-tools", + "plugin_id": "yaml.config.plugin", + "material": { + "type": "git", + "attributes": { + "url": "https://github.com/harisekhon/devops-python-tools", + "branch": "master", + "auto_update": true + } + }, + "configuration": [ + { + "key": "file_pattern", + "value": "*.gocd.y*ml" + } + ], + "rules": [ + { + "directive": "allow", + "action": "*", + "type": "*", + "resource": "*" + } + ] +} From 0cd3ced174bcb1e702648fad23b21f8851482618 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:08:35 +0000 Subject: [PATCH 1175/2779] added .gocd.yml --- .gocd.yml | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 .gocd.yml diff --git a/.gocd.yml b/.gocd.yml new file mode 100644 index 000000000..ad1c3d693 --- /dev/null +++ b/.gocd.yml @@ -0,0 +1,83 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://github.com/tomzo/gocd-yaml-config-plugin#setup + +# https://docs.gocd.org/current/configuration/configuration_reference.html + +format_version: 3 +pipelines: + devops-python-tools: + group: defaultGroup + label_template: ${COUNT} + lock_behavior: none + display_order: -1 + materials: + git: + git: https://github.com/harisekhon/devops-python-tools + shallow_clone: false + auto_update: true + branch: master + stages: + - build-and-test: + fetch_materials: true + keep_artifacts: false + clean_workspace: false + approval: + type: success + allow_only_on_success: false + jobs: + apt-update: + timeout: 10 + tasks: + - exec: + command: apt + arguments: + - update + run_if: passed + install-make: + timeout: 10 + tasks: + - exec: + command: apt + arguments: + - install + - -qy + - git + - make + run_if: passed + init: + timeout: 10 + tasks: + - exec: + command: make + arguments: + - init + run_if: passed + build: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - ci + run_if: passed + test: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - test + run_if: passed From 841b8126493d9ad1e334fac0213c54751a4c1807 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:08:57 +0000 Subject: [PATCH 1176/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3afe9935f..f5b5d1519 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Hari Sekhon - DevOps Python Tools [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) +[![GoCD](https://img.shields.io/badge/GoCD-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 544a5a4d4f0dbe53b66b4277a22cd5cfcee7a6a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:12:32 +0000 Subject: [PATCH 1177/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 16b436757..2f73faf5a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 16b4367579ff71056b3bdfb62b19550bf334c193 +Subproject commit 2f73faf5a9f2afc9c535858ed2c987c12b176306 From 2312e6d4a6759d494ffd1207f87877103ccda137 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 27 Mar 2020 22:12:32 +0000 Subject: [PATCH 1178/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index cd591772c..2ed6c0629 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit cd591772cb701831e207fa253aef56920ca7fc8d +Subproject commit 2ed6c06298ccd2421e8c8177d5ffa93321dc76ae From 1737491ca62dcfc03b35723de122c8217f412aba Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 28 Mar 2020 11:25:24 +0000 Subject: [PATCH 1179/2779] updated cloudera_navigator_tables_used_postgres.py --- cloudera_navigator_tables_used_postgres.py | 55 +++++++++++----------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py index 7d2fcaee9..47bf1b419 100755 --- a/cloudera_navigator_tables_used_postgres.py +++ b/cloudera_navigator_tables_used_postgres.py @@ -49,7 +49,6 @@ srcdir = os.path.abspath(os.path.dirname(__file__)) pylib = os.path.join(srcdir, 'pylib') lib = os.path.join(srcdir, 'lib') -import gzip sys.path.append(pylib) sys.path.append(lib) try: @@ -64,7 +63,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.2.1' class ClouderaNavigatorTablesUsedPostgreSQL(ClouderaNavigatorTablesUsed): @@ -122,29 +121,29 @@ def process_file(self, filehandle): self.len_headers = len(headers) # needed to ensure row joining works later on with number of fields left assert self.len_headers == 14 or self.len_headers == 16 - user_index = 4 - assert headers[user_index] == 'username' - # Hive postgres audit log - if self.len_headers == 14: - operation_index = 6 - database_index = 7 - table_index = 9 - sql_index = 10 - resource_index = 12 - assert headers[resource_index] == 'resource_path' - # Impala postgres audit log - elif self.len_headers == 16: - operation_index = 7 - database_index = 11 - table_index = 13 - sql_index = 15 - resource_index = None - else: - raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala') - assert headers[sql_index] == 'operation_text' - assert headers[database_index] == 'database_name' - assert headers[table_index] == 'table_name' - assert headers[operation_index] == 'operation' + user_index = 4 + assert headers[user_index] == 'username' + # Hive postgres audit log + if self.len_headers == 14: + operation_index = 6 + database_index = 7 + table_index = 9 + sql_index = 10 + resource_index = 12 + assert headers[resource_index] == 'resource_path' + # Impala postgres audit log + elif self.len_headers == 16: + operation_index = 7 + database_index = 11 + table_index = 13 + sql_index = 15 + resource_index = None + else: + raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala') + assert headers[sql_index] == 'operation_text' + assert headers[database_index] == 'database_name' + assert headers[table_index] == 'table_name' + assert headers[operation_index] == 'operation' self.indicies = { 'user_index': user_index, 'operation_index': operation_index, @@ -215,9 +214,9 @@ def parse_table(self, row): return (database, table) def get_db_table_from_resource(self, row): - # only available for hive audit logs, not impala - if self.indicies['resource_index'] is None: - return (None, None) + # only available for hive audit logs, not impala + if self.indicies['resource_index'] is None: + return (None, None) database = None table = None resource = row[self.indicies['resource_index']] From e3979c89f52b16fd955c6d09b20d24bcd904fae3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 28 Mar 2020 11:25:31 +0000 Subject: [PATCH 1180/2779] updated .appveyor.yml --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c1e4e06ac..a97f97008 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -28,10 +28,10 @@ install: # The following packages have unmet dependencies: # mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed # E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. -# bash-tools/Makefile.in:272: recipe for target 'apt-packages' failed +# devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed # make[2]: *** [apt-packages] Error 123 # make[2]: Leaving directory '/home/appveyor/projects/pylib' -# bash-tools/Makefile.in:212: recipe for target 'system-packages' failed +# devops-python-tools/Makefile.in:212: recipe for target 'system-packages' failed # # adding "|| :" to the end of these commands causes them to be silently ignored! - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list From 50b24cbf567353f2691ac3ef3d15c361cfb4fe80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 28 Mar 2020 11:25:35 +0000 Subject: [PATCH 1181/2779] updated shippable.yml --- shippable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shippable.yml b/shippable.yml index 6b45ef91f..43f564753 100644 --- a/shippable.yml +++ b/shippable.yml @@ -29,7 +29,7 @@ build: # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed. - # bash-tools/Makefile.in:272: recipe for target 'apt-packages' failed + # devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed - rm -fv /etc/apt/sources.list.d/cassandra.sources.list* - rm -fv /etc/apt/sources.list.d/yarn.list* #- shippable_retry make From 8241e557a6315ec6d313374938140ab86d2b1578 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 04:55:57 +0100 Subject: [PATCH 1182/2779] added Jenkinsfile --- Jenkinsfile | 134 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..0c1855ec4 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,134 @@ +#!/usr/bin/env groovy +// vim:ts=4:sts=4:sw=4:et:filetype=groovy:syntax=groovy +// +// Author: Hari Sekhon +// Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017) +// +// https://github.com/harisekhon/devops-python-tools +// +// License: see accompanying Hari Sekhon LICENSE file +// +// If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +// +// https://www.linkedin.com/in/harisekhon +// + +// ========================================================================== // +// J e n k i n s P i p e l i n e +// ========================================================================== // + + +// https://jenkins.io/doc/book/pipeline/syntax/ + + +pipeline { + // run pipeline any agent + agent any + // can't do this when running jenkins in docker itself, gets '.../script.sh: docker: not found' +// agent { +// docker { +// image 'ubuntu:18.04' +// args '-v $HOME/.m2:/root/.m2 -v $HOME/.cache/pip:/root/.cache/pip -v $HOME/.cpanm:/root/.cpanm -v $HOME/.sbt:/root/.sbt -v $HOME/.ivy2:/root/.ivy2 -v $HOME/.gradle:/root/.gradle' +// } +// } + + // need to specify at least one env var if enabling + //environment { + // DEBUG = '1' + //} + + options { + // put timestamps in console logs + timestamps() + + // timeout entire pipeline after 4 hours + timeout(time: 4, unit: 'HOURS') + + //retry entire pipeline 3 times + //retry(3) + } + + triggers { + cron('H 10 * * 1-5') + pollSCM('H/2 * * * *') + } + + stages { + stage ('Checkout') { + steps { + checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]]) + } + } + + stage('Build') { + steps { + echo "Running ${env.BUILD_ID} on ${env.JENKINS_URL}" + echo 'Building...' + timeout(time: 10, unit: 'MINUTES') { + retry(3) { +// sh 'apt update -q' +// sh 'apt install -qy make' +// sh 'make init' + sh """ + apt update -q && + apt install -qy make && + make init + """ + } + } + timeout(time: 180, unit: 'MINUTES') { + sh 'make ci' + } + } + } + + stage('Test') { + options { + retry(2) + } + steps { + echo 'Testing...' + timeout(time: 120, unit: 'MINUTES') { + sh 'make test' + } + } + } + +// stage('Human gate') { +// steps { +// input "Proceed to deployment?" +// } +// } +// +// stage('Deployment') { +// steps { +// echo 'Deploying...' +// echo 'Nothing to deploy' +// } +// } + + } + post { + always { + echo 'Always' + //deleteDir() // clean up workspace + + // collect JUnit reports for Jenkins UI + //junit 'build/reports/**/*.xml' + // collect artifacts to Jenkins for analysis + //archiveArtifacts artifacts: 'build/libs/**/*.jar', fingerprint: true + } + success { + echo 'SUCCESS!' + } + failure { + echo 'FAILURE!' + } + unstable { + echo 'UNSTABLE!' + } + changed { + echo 'Pipeline state change! (success vs failure)' + } + } +} From 3be0fa7b8d271587c5eed03bd72d6b2d251202fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 04:55:59 +0100 Subject: [PATCH 1183/2779] added jenkins-job.xml --- setup/jenkins-job.xml | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 setup/jenkins-job.xml diff --git a/setup/jenkins-job.xml b/setup/jenkins-job.xml new file mode 100644 index 000000000..4469d2afe --- /dev/null +++ b/setup/jenkins-job.xml @@ -0,0 +1,57 @@ + + + + + + + + hudson.triggers.SCMTrigger + hudson.triggers.TimerTrigger + + + + + + + false + + + + https://github.com/harisekhon/devops-python-tools/ + + + + + + H 10 * * 1-5 + + + H/2 * * * * + false + + + + + + + 2 + + + https://github.com/harisekhon/devops-python-tools + + + + + */master + + + false + + + + Jenkinsfile + true + + + false + \ No newline at end of file From a51f3b0ec279bb9549c6c791a02583d5833ee34c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 05:02:13 +0100 Subject: [PATCH 1184/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f5b5d1519..380e9e7cd 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Hari Sekhon - DevOps Python Tools [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) +[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From f4e31a54a4626d54078d5d2f7d96839215f48acb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 05:03:13 +0100 Subject: [PATCH 1185/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2f73faf5a..450fe18ff 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2f73faf5a9f2afc9c535858ed2c987c12b176306 +Subproject commit 450fe18ff0341e348235645bb6cdaedcee2c7b00 From 47388a28394faed0b4dfd9f2678212ae51743069 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 29 Mar 2020 05:03:13 +0100 Subject: [PATCH 1186/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2ed6c0629..22de6ec06 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2ed6c06298ccd2421e8c8177d5ffa93321dc76ae +Subproject commit 22de6ec0636ddbb61e4b710d2d32a0a4c6a37775 From 6c11f3653036719d2c11598f0136c61d6cc70968 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:17:00 +0100 Subject: [PATCH 1187/2779] updated Jenkinsfile --- Jenkinsfile | 56 ++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0c1855ec4..c63e3e540 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,35 +22,35 @@ pipeline { - // run pipeline any agent + // run pipeline any agent agent any - // can't do this when running jenkins in docker itself, gets '.../script.sh: docker: not found' + // can't do this when running jenkins in docker itself, gets '.../script.sh: docker: not found' // agent { -// docker { -// image 'ubuntu:18.04' -// args '-v $HOME/.m2:/root/.m2 -v $HOME/.cache/pip:/root/.cache/pip -v $HOME/.cpanm:/root/.cpanm -v $HOME/.sbt:/root/.sbt -v $HOME/.ivy2:/root/.ivy2 -v $HOME/.gradle:/root/.gradle' -// } -// } +// docker { +// image 'ubuntu:18.04' +// args '-v $HOME/.m2:/root/.m2 -v $HOME/.cache/pip:/root/.cache/pip -v $HOME/.cpanm:/root/.cpanm -v $HOME/.sbt:/root/.sbt -v $HOME/.ivy2:/root/.ivy2 -v $HOME/.gradle:/root/.gradle' +// } +// } - // need to specify at least one env var if enabling + // need to specify at least one env var if enabling //environment { - // DEBUG = '1' - //} + // DEBUG = '1' + //} options { // put timestamps in console logs timestamps() - // timeout entire pipeline after 4 hours - timeout(time: 4, unit: 'HOURS') + // timeout entire pipeline after 4 hours + timeout(time: 4, unit: 'HOURS') - //retry entire pipeline 3 times - //retry(3) + //retry entire pipeline 3 times + //retry(3) } - triggers { + triggers { cron('H 10 * * 1-5') - pollSCM('H/2 * * * *') + pollSCM('H/2 * * * *') } stages { @@ -69,11 +69,11 @@ pipeline { // sh 'apt update -q' // sh 'apt install -qy make' // sh 'make init' - sh """ - apt update -q && - apt install -qy make && - make init - """ + sh """ + apt update -q && + apt install -qy make && + make init + """ } } timeout(time: 180, unit: 'MINUTES') { @@ -103,20 +103,20 @@ pipeline { // stage('Deployment') { // steps { // echo 'Deploying...' -// echo 'Nothing to deploy' +// echo 'Nothing to deploy' // } // } } - post { + post { always { echo 'Always' - //deleteDir() // clean up workspace + //deleteDir() // clean up workspace - // collect JUnit reports for Jenkins UI - //junit 'build/reports/**/*.xml' - // collect artifacts to Jenkins for analysis - //archiveArtifacts artifacts: 'build/libs/**/*.jar', fingerprint: true + // collect JUnit reports for Jenkins UI + //junit 'build/reports/**/*.xml' + // collect artifacts to Jenkins for analysis + //archiveArtifacts artifacts: 'build/libs/**/*.jar', fingerprint: true } success { echo 'SUCCESS!' From baf63b3b5affb9ae1614a47893df298ed015dbb9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:17:44 +0100 Subject: [PATCH 1188/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 450fe18ff..3b044f028 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 450fe18ff0341e348235645bb6cdaedcee2c7b00 +Subproject commit 3b044f02840047b21931ecb54ab50e72fcf51efd From c7b27780b3a6a2adc02e3279a3dcb70968ea67c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:17:44 +0100 Subject: [PATCH 1189/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 22de6ec06..ac2c5362a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 22de6ec0636ddbb61e4b710d2d32a0a4c6a37775 +Subproject commit ac2c5362a27e7f77f823a60bdfb6e3277b32b66d From d9f2c4e90dd4d247913e060bf814da330ba8c1ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:25:42 +0100 Subject: [PATCH 1190/2779] updated rpm-packages-optional.txt --- setup/rpm-packages-optional.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt index 11e757a6f..cb889fb12 100644 --- a/setup/rpm-packages-optional.txt +++ b/setup/rpm-packages-optional.txt @@ -19,3 +19,9 @@ yamllint snappy-devel # CentOS 8 csnappy-devel + +# postgres pg_config +# CentOS 8 +libpq-devel +# CentOS 7 +postgresql-devel From 5378204d20bd4b0bd7e675a89f8e2d9b43efca9b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:25:50 +0100 Subject: [PATCH 1191/2779] updated rpm-packages-dev.txt --- setup/rpm-packages-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index 2537f9e1a..6dd8fa477 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -17,7 +17,6 @@ gcc-c++ # needed to build python-krbV and cloudera/thrift_sasl cyrus-sasl-devel krb5-devel -libpq-devel # postgres pg_config openldap-devel openssl-devel From 6a5de18e631bb58b42468b3f4efc61557e2e1074 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:33:37 +0100 Subject: [PATCH 1192/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3b044f028..19b964eef 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3b044f02840047b21931ecb54ab50e72fcf51efd +Subproject commit 19b964eef3f2d6a95f13ca8bfc7b4b598e55d1ea From f4b325e0ab68cd420f32720460a3b3c8a46c61b9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:33:37 +0100 Subject: [PATCH 1193/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ac2c5362a..289fce9b5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ac2c5362a27e7f77f823a60bdfb6e3277b32b66d +Subproject commit 289fce9b55e7d8464f62d20e106dc88f71165e03 From 9d427b5dce3393bc1586bc12e2aee416b84911da Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 11:35:18 +0100 Subject: [PATCH 1194/2779] updated README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 380e9e7cd..e9a8ab490 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,11 @@ Hari Sekhon - DevOps Python Tools [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) --> [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey)](https://github.com/HariSekhon/DevOps-Python-tools) + + [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) From 5a124d955410c1b84053a27806d0645a3d5f1488 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 14:49:58 +0100 Subject: [PATCH 1195/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 19b964eef..079f5cd34 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 19b964eef3f2d6a95f13ca8bfc7b4b598e55d1ea +Subproject commit 079f5cd34b88c6a8c6f07bcd011e68fee5879ea0 From 11bc05c086ab03d65fd3ccac22aa8fb9351911f2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 14:49:58 +0100 Subject: [PATCH 1196/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 289fce9b5..9974c54ca 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 289fce9b55e7d8464f62d20e106dc88f71165e03 +Subproject commit 9974c54cab38f6d9f8b5250a6f02f50e7e058444 From 99ae9d0468803c7eab00dedc3e0448c77e035b9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:01:36 +0100 Subject: [PATCH 1197/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 079f5cd34..d29d724e2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 079f5cd34b88c6a8c6f07bcd011e68fee5879ea0 +Subproject commit d29d724e230fb3403982b805f29dd22ea807e3d3 From c4a70bdb0912043290d8ec9b229240fdd97cf6e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:01:36 +0100 Subject: [PATCH 1198/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9974c54ca..8bd7ad714 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9974c54cab38f6d9f8b5250a6f02f50e7e058444 +Subproject commit 8bd7ad714a45d19d44ebe40eaf4bb6bd8b426d06 From 853b054feedceda1ba950422b0bf26d2d895fa0d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:40:54 +0100 Subject: [PATCH 1199/2779] updated Makefile --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index c3689b47e..e9823683c 100755 --- a/Makefile +++ b/Makefile @@ -57,6 +57,8 @@ build: @echo ========================= @echo DevOps Python Tools Build @echo ========================= + @bash-tools/git_summary_line.sh + @echo @# executing in sh where type is not available @#type -P python From a31c1c90be15b08c05cc1c9509e9c9bf8c178da1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:41:03 +0100 Subject: [PATCH 1200/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d29d724e2..100b4e6c6 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d29d724e230fb3403982b805f29dd22ea807e3d3 +Subproject commit 100b4e6c65d877e136df18b484e2f7848b9f4c45 From aa29b6641713c397e2be06d4b8c73abe5f57394e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 19:41:03 +0100 Subject: [PATCH 1201/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8bd7ad714..c264da75c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8bd7ad714a45d19d44ebe40eaf4bb6bd8b426d06 +Subproject commit c264da75cfd484753a712fece7acc1405e1b8309 From 0387479ed6d3f0b2138bd2b928d3687a05aac23a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 20:01:34 +0100 Subject: [PATCH 1202/2779] updated Makefile --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e9823683c..3b5d29391 100755 --- a/Makefile +++ b/Makefile @@ -53,11 +53,11 @@ ifndef SKIP_PARQUET endif .PHONY: build -build: +build: init @echo ========================= @echo DevOps Python Tools Build @echo ========================= - @bash-tools/git_summary_line.sh + @$(MAKE) git-summary @echo @# executing in sh where type is not available @@ -66,7 +66,6 @@ build: python -V || : pip -V || : - $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python if type apk 2>/dev/null; then $(MAKE) apk-packages-extra; fi From 1d102fc1bfb6fc821e054c0599b6e44016849be1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 20:01:50 +0100 Subject: [PATCH 1203/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 100b4e6c6..f60229172 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 100b4e6c65d877e136df18b484e2f7848b9f4c45 +Subproject commit f60229172e123257e3798549348c9079591cbe0e From 55215d0a8ce06ebc3940d6c70a6bfab8513ee33b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 20:01:50 +0100 Subject: [PATCH 1204/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c264da75c..4e7bbd2ac 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c264da75cfd484753a712fece7acc1405e1b8309 +Subproject commit 4e7bbd2ace765b28deef6e8b20ec3c76680190e2 From 2d1023350d9d02d0784a00cb1f05984824ea5973 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 20:02:33 +0100 Subject: [PATCH 1205/2779] updated Makefile --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 3b5d29391..4eb3ff5e1 100755 --- a/Makefile +++ b/Makefile @@ -58,7 +58,6 @@ build: init @echo DevOps Python Tools Build @echo ========================= @$(MAKE) git-summary - @echo @# executing in sh where type is not available @#type -P python From 1132d7c8f61c7743584e29fbdbaaa9e2741f6868 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 21:44:01 +0100 Subject: [PATCH 1206/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f60229172..80a090777 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f60229172e123257e3798549348c9079591cbe0e +Subproject commit 80a090777285e0b22c1225756a2e07323625b8a3 From 5055dd80fc33380a62a09547b9e537f9ed835868 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 31 Mar 2020 21:44:01 +0100 Subject: [PATCH 1207/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4e7bbd2ac..5c9b49fbe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4e7bbd2ace765b28deef6e8b20ec3c76680190e2 +Subproject commit 5c9b49fbe3e28f606fb6dc62e023894fa9cae236 From 829cde2bb53a92d3bf7dc698c78f3cda21f64782 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 1 Apr 2020 14:03:58 +0100 Subject: [PATCH 1208/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 80a090777..167c12679 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 80a090777285e0b22c1225756a2e07323625b8a3 +Subproject commit 167c12679aa4c5779e989c9bfc41ef605ff1ab3f From 2eb77b0e1662f73e3feeaf82d3cabd40f17ae17a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 1 Apr 2020 14:03:58 +0100 Subject: [PATCH 1209/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5c9b49fbe..aead6c401 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5c9b49fbe3e28f606fb6dc62e023894fa9cae236 +Subproject commit aead6c401b0c7ec92366eee33cce338f8eb6a8cb From c2cb9ea9f428f2ed275e3699b35c9beb0bdcc119 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 16:06:20 +0100 Subject: [PATCH 1210/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 167c12679..d67dcede2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 167c12679aa4c5779e989c9bfc41ef605ff1ab3f +Subproject commit d67dcede2bbff47072b655ee641357723d91cc0f From 5e2e95b0753c5c85e0e2a874fbaa8e20c51b65d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 16:06:20 +0100 Subject: [PATCH 1211/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index aead6c401..0d1827a52 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit aead6c401b0c7ec92366eee33cce338f8eb6a8cb +Subproject commit 0d1827a52e68a4174e8c1059971e884e2c14cb67 From a6643ab4f5fe3e09b9cf34b8df60565f5a05cf5a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 16:31:57 +0100 Subject: [PATCH 1212/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e9a8ab490..0ffbe1aa5 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) -[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STATUS.md) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) From 65a40cd1b1b4e0f82ee958970ce98f3b78d709d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 19:05:13 +0100 Subject: [PATCH 1213/2779] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0ffbe1aa5..4490e2f81 100644 --- a/README.md +++ b/README.md @@ -441,3 +441,5 @@ You might also be interested in the following really nice Jupyter notebook for H ### Stargazers over time [![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) + +[git.io/python-tools](https://git.io/python-tools) From 2c94c24a611b78b990d237165fd7e1937d9defa3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 19:49:27 +0100 Subject: [PATCH 1214/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d67dcede2..7294e61f1 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d67dcede2bbff47072b655ee641357723d91cc0f +Subproject commit 7294e61f1af8b2cf0a60b670136635b6d417fd5d From 1748e1da2fa62a36f8a65b2454aaf7b37bff1ef6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 19:49:27 +0100 Subject: [PATCH 1215/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0d1827a52..175ffba88 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0d1827a52e68a4174e8c1059971e884e2c14cb67 +Subproject commit 175ffba88fc740f0dbaf211d2675770affe5e382 From 9aecf1066d1be7186636318927a4c62feefdd4c6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 22:25:51 +0100 Subject: [PATCH 1216/2779] updated schedule to 7am daily --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos6.yaml.disabled | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml | 2 +- .github/workflows/debian_9.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_10.15.yaml | 2 +- .github/workflows/python2.yaml | 2 +- .github/workflows/python3.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index b4ea64cf0..6d7345ccb 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index b9d1305a4..db677d27e 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 13ff19170..1d98a2d0b 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/centos6.yaml.disabled b/.github/workflows/centos6.yaml.disabled index 6ec523536..dbbadfee1 100644 --- a/.github/workflows/centos6.yaml.disabled +++ b/.github/workflows/centos6.yaml.disabled @@ -24,7 +24,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 7ebef0745..0694d60a1 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 3868ce406..2751546a5 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 9969fb0a1..635668ca1 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 7980bae04..9fc388826 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index f0ec8860c..96673f75d 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 640d28a79..622715687 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index a7acf16cc..ca5afbfa5 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index e03e57161..0cfedafc6 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 1f7100849..e659a673d 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 0c1358164..87bbf2065 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index a16b0731f..22dfce874 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/python2.yaml b/.github/workflows/python2.yaml index e9489be32..c13d47aa4 100644 --- a/.github/workflows/python2.yaml +++ b/.github/workflows/python2.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/python3.yaml b/.github/workflows/python3.yaml index 16932c744..b6304c6e1 100644 --- a/.github/workflows/python3.yaml +++ b/.github/workflows/python3.yaml @@ -22,7 +22,7 @@ on: - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 840e5553a..313177077 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index f04e0bfda..29bb85bc9 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index c122a9cab..0795ff15c 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index ff17f01cd..d607fe25c 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -22,7 +22,7 @@ on: # [push] - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 10 30 * *' + - cron: '0 7 * * *' jobs: build: From 9b2b4f86642726503e954810c8ac8f0bc2f895ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 22:26:00 +0100 Subject: [PATCH 1217/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7294e61f1..31cb96f72 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7294e61f1af8b2cf0a60b670136635b6d417fd5d +Subproject commit 31cb96f729fe0f83d6f5b6845a09d98b8d268ffa From dd152672d35ffb7bffbd0487433732d5ce8f85f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 2 Apr 2020 22:26:00 +0100 Subject: [PATCH 1218/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 175ffba88..b99613841 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 175ffba88fc740f0dbaf211d2675770affe5e382 +Subproject commit b99613841ba4e867bd0eaa565efdfd45230a77e5 From 459d87ad391ed11dcc2b85a7db1b41e02c7f726a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:03:58 +0100 Subject: [PATCH 1219/2779] updated .appveyor.yml --- .appveyor.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index a97f97008..9d75f5a37 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -17,6 +17,14 @@ image: Ubuntu +# https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ +environment: + APPVEYOR_SSH_KEY: AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== + +# enable SSH session accessible via my public key +init: + - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + install: # workaround for: # Some packages could not be installed. This may mean that you have From 1d979b780b19068ed2a72cc0e1e300c722271df5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:32:31 +0100 Subject: [PATCH 1220/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4490e2f81..80d0921dc 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Hari Sekhon - DevOps Python Tools [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) -[![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/DevOps-Python-tools/builds) +[![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) [![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) From 59639c3a6fb1cbd43af334028854a2861cb2f420 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:41:59 +0100 Subject: [PATCH 1221/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80d0921dc..f1f3c9027 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Hari Sekhon - DevOps Python Tools [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) -[![BuildKite](https://img.shields.io/buildkite/314d5913c332d6f1eebad4a10f23da906bca544bdde6550595/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) +[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) From fc52b36cc3d399d7c7919f85dd8e30ee2a8285a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:43:48 +0100 Subject: [PATCH 1222/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 31cb96f72..e78ac3856 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 31cb96f729fe0f83d6f5b6845a09d98b8d268ffa +Subproject commit e78ac38568d3cb6702da5e58f543d5d531653738 From d91a578eec4bd5608923e752ec5c4a29ada898a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 16:43:48 +0100 Subject: [PATCH 1223/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b99613841..ae3b05c6d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b99613841ba4e867bd0eaa565efdfd45230a77e5 +Subproject commit ae3b05c6da98d8f086359b0d5b630c39e72f9845 From 67a5eca57863a64ca84b7cdd0f7cbaac6895115a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 18:42:46 +0100 Subject: [PATCH 1224/2779] updated .appveyor.yml --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 9d75f5a37..ddf4ecdf9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -19,7 +19,7 @@ image: Ubuntu # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ environment: - APPVEYOR_SSH_KEY: AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== + APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension # enable SSH session accessible via my public key init: From 5b93d80aa0b75687e92ca612b758f0db161cba7f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 18:59:51 +0100 Subject: [PATCH 1225/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e78ac3856..8333adc0c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e78ac38568d3cb6702da5e58f543d5d531653738 +Subproject commit 8333adc0cf42adcc4dc890c62ceb05afd352ea4c From 5cfe538682abf54e615976632fa67d4fc717a379 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 18:59:51 +0100 Subject: [PATCH 1226/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ae3b05c6d..ac6ba0d8a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ae3b05c6da98d8f086359b0d5b630c39e72f9845 +Subproject commit ac6ba0d8a46c5ae84dedca563eed06a23ff58bbe From d4e7d57f95d02fb27bdc177ffc66e916e21d6d99 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 19:18:56 +0100 Subject: [PATCH 1227/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8333adc0c..57937e3f4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8333adc0cf42adcc4dc890c62ceb05afd352ea4c +Subproject commit 57937e3f4d20f139daeaa1236fa47d9a20243da6 From f7f1f6adb3ff5e50fe3ee2e481242973d22fcb55 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 4 Apr 2020 19:18:56 +0100 Subject: [PATCH 1228/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ac6ba0d8a..0f144150e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ac6ba0d8a46c5ae84dedca563eed06a23ff58bbe +Subproject commit 0f144150ef41f84b7ca95dd59d7e64136df0ccf3 From 8656c125b4413f075f9f2d8c5bfe926237ded15b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 5 Apr 2020 13:57:39 +0100 Subject: [PATCH 1229/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 57937e3f4..7539c4267 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 57937e3f4d20f139daeaa1236fa47d9a20243da6 +Subproject commit 7539c426766c44bbfdff1480fd15a46689d6745d From 5a6a5908c298bc7b4be045a8b39a09a50590b14b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 5 Apr 2020 13:57:39 +0100 Subject: [PATCH 1230/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0f144150e..114c7a54d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0f144150ef41f84b7ca95dd59d7e64136df0ccf3 +Subproject commit 114c7a54d217b5b5eb729ccb4e5d8fe3ec6ca266 From c7da2a0087f7b86f576eea0e18730d968e4eb783 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 5 Apr 2020 21:17:54 +0100 Subject: [PATCH 1231/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 114c7a54d..57b868bf0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 114c7a54d217b5b5eb729ccb4e5d8fe3ec6ca266 +Subproject commit 57b868bf096cd8574660becff7b9006996f48ec7 From 2742843fe5a016f49559c68898de2ef1d65d014c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Apr 2020 23:16:25 +0100 Subject: [PATCH 1232/2779] updated test_find_active_server.sh --- tests/test_find_active_server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_find_active_server.sh b/tests/test_find_active_server.sh index 1bde9e0e6..03e8dee75 100755 --- a/tests/test_find_active_server.sh +++ b/tests/test_find_active_server.sh @@ -144,7 +144,7 @@ ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_server.py --https local echo "testing https with url path and regex matching:" echo -run_grep "^github.com$" ./find_active_server.py $opts --https $WEBSITE2 github.com -u /harisekhon --regex 'python-tools' +run_grep "^github.com$" ./find_active_server.py $opts --https $WEBSITE2 github.com -u /harisekhon --regex '(?i)python-tools' # ============================================================================ # From 3d255727c176193fadb634ab87f72d4a3d2a661f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Apr 2020 23:16:40 +0100 Subject: [PATCH 1233/2779] changed cpu_count import to work with Python 3 --- find_active_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/find_active_server.py b/find_active_server.py index edcf0fb49..dea660dbb 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -112,7 +112,8 @@ import subprocess import sys #from threading import Thread -from multiprocessing.pool import ThreadPool, cpu_count +from multiprocessing.pool import ThreadPool +from multiprocessing import cpu_count # prefer blocking semantics of que.get() rather than handling deque.popleft() => 'IndexError: pop from an empty deque' #from collections import deque import Queue @@ -137,7 +138,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.4' +__version__ = '0.8.5' class FindActiveServer(CLI): From f0407797c76f1b91d8cde1fda25a5b34a4fc7d56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Apr 2020 23:23:58 +0100 Subject: [PATCH 1234/2779] updated Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c63e3e540..99d0ac0b7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -103,7 +103,7 @@ pipeline { // stage('Deployment') { // steps { // echo 'Deploying...' -// echo 'Nothing to deploy' +// echo 'Nothing to deploy' // } // } From 3f5a2605bf5d40a2cc6c1aa311406b6eae242f79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 6 Apr 2020 23:41:26 +0100 Subject: [PATCH 1235/2779] updated .appveyor.yml --- .appveyor.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index ddf4ecdf9..0bce84c3e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -22,7 +22,13 @@ environment: APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension # enable SSH session accessible via my public key -init: +#init: +# - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + +# more useful at end to let .appveyor's tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to check out code and do fixes manually +on_finish: + # set this in Settings -> Environment dynamically instead of here + #- sh: export APPVEYOR_SSH_BLOCK=true - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - install: From 3ded1883c6adf0c799f35a71ff141ead14850d6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 00:38:36 +0100 Subject: [PATCH 1236/2779] more Python 3 fixes --- find_active_server.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/find_active_server.py b/find_active_server.py index dea660dbb..f759527eb 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -116,9 +116,13 @@ from multiprocessing import cpu_count # prefer blocking semantics of que.get() rather than handling deque.popleft() => 'IndexError: pop from an empty deque' #from collections import deque -import Queue import traceback from random import shuffle +# Python 2 Queue vs Python 3 queue module :-/ +if sys.version[0] == '2': + import Queue as queue +else: + import queue as queue try: import requests except ImportError: @@ -138,7 +142,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.5' +__version__ = '0.8.6' class FindActiveServer(CLI): @@ -157,7 +161,7 @@ def __init__(self): self.request_timeout = None self.default_num_threads = min(cpu_count() * 4, 100) self.num_threads = None - self.queue = Queue.Queue() + self.queue = queue.Queue() self.pool = None def add_options(self): From 8a0087bd2e0e43f761a69e77eb575881eec98596 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 00:55:51 +0100 Subject: [PATCH 1237/2779] updated .appveyor.yml --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 0bce84c3e..7934f0253 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -25,7 +25,7 @@ environment: #init: # - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - -# more useful at end to let .appveyor's tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to check out code and do fixes manually +# more useful at end to leverage .appveyor.yml tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to do all that manually in SSH session on_finish: # set this in Settings -> Environment dynamically instead of here #- sh: export APPVEYOR_SSH_BLOCK=true From be1f1f2698ae1a55d7bec083d92ea3fb8e22e266 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 00:56:50 +0100 Subject: [PATCH 1238/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7539c4267..a708b2b76 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7539c426766c44bbfdff1480fd15a46689d6745d +Subproject commit a708b2b769ec3ddafedde7dbe8d02a7899b5df90 From 8b4521483b5d59a8b4908a85b65b74e8858bc341 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 00:56:50 +0100 Subject: [PATCH 1239/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 57b868bf0..85f863007 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 57b868bf096cd8574660becff7b9006996f48ec7 +Subproject commit 85f863007a5259e077875b660248c69035410b2d From 949ec04f975f3e75168a60373a1cf8fa70a9212a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 12:06:25 +0100 Subject: [PATCH 1240/2779] renamed .github/workflows/python2.yaml to .github/workflows/python2.6.yaml --- .github/workflows/{python2.yaml => python2.6.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python2.yaml => python2.6.yaml} (100%) diff --git a/.github/workflows/python2.yaml b/.github/workflows/python2.6.yaml similarity index 100% rename from .github/workflows/python2.yaml rename to .github/workflows/python2.6.yaml From 211f6e038a004f135036dc10038fc1cb705cf9d5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 12:06:32 +0100 Subject: [PATCH 1241/2779] renamed .github/workflows/python3.yaml to .github/workflows/python3.6.yaml --- .github/workflows/{python3.yaml => python3.6.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python3.yaml => python3.6.yaml} (100%) diff --git a/.github/workflows/python3.yaml b/.github/workflows/python3.6.yaml similarity index 100% rename from .github/workflows/python3.yaml rename to .github/workflows/python3.6.yaml From 51195606367f1c619ed4eb28ae6cbad1e3d42500 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 12:10:44 +0100 Subject: [PATCH 1242/2779] renamed .github/workflows/python2.6.yaml to .github/workflows/python2.7.yaml --- .github/workflows/{python2.6.yaml => python2.7.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python2.6.yaml => python2.7.yaml} (100%) diff --git a/.github/workflows/python2.6.yaml b/.github/workflows/python2.7.yaml similarity index 100% rename from .github/workflows/python2.6.yaml rename to .github/workflows/python2.7.yaml From eeb2fe8b43465c5f975483e6638b8120f6f35e2e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:30:39 +0100 Subject: [PATCH 1243/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a708b2b76..9731e35b3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a708b2b769ec3ddafedde7dbe8d02a7899b5df90 +Subproject commit 9731e35b3bfeaece3b9706e3b2ec6f232041e22b From b3cf3887a8de79bf9d7e9da96b625897dc3c5ff9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:30:39 +0100 Subject: [PATCH 1244/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 85f863007..9d38d37af 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 85f863007a5259e077875b660248c69035410b2d +Subproject commit 9d38d37afeb88a09095cd04f5abd3d9e1ae8230b From a3cec30cf60e9527c9a8a311bec887f8fbf66e57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:31:11 +0100 Subject: [PATCH 1245/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index c13d47aa4..89e33295e 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -27,12 +27,12 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest] - #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] python-version: [2.7] steps: - uses: actions/checkout@v2 @@ -42,8 +42,12 @@ jobs: - uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} restore-keys: | - ${{ runner.os }}-pip- - - name: build & test - run: make build test + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 9c325dc77e0d6d89b031b784942f80cd7328a18e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:31:18 +0100 Subject: [PATCH 1246/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index b6304c6e1..704b99878 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -27,12 +27,12 @@ on: jobs: build: #name: build - timeout-minutes: 10 + timeout-minutes: 60 runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest] - #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] python-version: [3.6] steps: - uses: actions/checkout@v2 @@ -42,8 +42,12 @@ jobs: - uses: actions/cache@v1 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} restore-keys: | - ${{ runner.os }}-pip- - - name: build & test - run: make build test + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 9458bb27559a8f2e3ff1627cf86d7886e52a4240 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:31:35 +0100 Subject: [PATCH 1247/2779] updated mac.yaml --- .github/workflows/mac.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 87bbf2065..eb7a8ce81 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -39,7 +39,9 @@ jobs: ${{ runner.os }}-pip-devops-python-tools - name: brew update run: which brew && brew update || echo + - name: init + run: make init - name: build - run: make + run: make ci - name: test run: make test From 7617192bbca7d49bae31f3025409cd1495083914 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:31:48 +0100 Subject: [PATCH 1248/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 22dfce874..bc8004be0 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -39,7 +39,9 @@ jobs: ${{ runner.os }}-pip-devops-python-tools - name: brew update run: which brew && brew update || echo + - name: init + run: make init - name: build - run: make + run: make ci - name: test run: make test From 21bff4b4cc1b6bdeeac7409c6f17ccba82840cda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:04 +0100 Subject: [PATCH 1249/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 313177077..c96960027 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -37,7 +37,9 @@ jobs: key: ${{ runner.os }}-pip-devops-python-tools # ${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init - name: build - run: make + run: make ci - name: test run: make test From ef7274b6aa73537de967160683c5b1f8359bbe3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:20 +0100 Subject: [PATCH 1250/2779] added python3.5.yaml --- .github/workflows/python3.5.yaml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/python3.5.yaml diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml new file mode 100644 index 000000000..79856bbc1 --- /dev/null +++ b/.github/workflows/python3.5.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 3.5 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [3.5] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From d436e331446f40e7f9b08b157e77bc9e11df965b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:21 +0100 Subject: [PATCH 1251/2779] added python3.7.yaml --- .github/workflows/python3.7.yaml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/python3.7.yaml diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml new file mode 100644 index 000000000..538144130 --- /dev/null +++ b/.github/workflows/python3.7.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 3.7 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [3.7] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 018b8964d042e9b1d1915c851d91666b9f0a2df8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:23 +0100 Subject: [PATCH 1252/2779] added python3.8.yaml --- .github/workflows/python3.8.yaml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/python3.8.yaml diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml new file mode 100644 index 000000000..6f1c69d4f --- /dev/null +++ b/.github/workflows/python3.8.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Python 3.8 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [3.8] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From ee8f045828f596bc71946c3655be829aa8d38186 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:25 +0100 Subject: [PATCH 1253/2779] added pypy2.yaml --- .github/workflows/pypy2.yaml | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/pypy2.yaml diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml new file mode 100644 index 000000000..28a8a6445 --- /dev/null +++ b/.github/workflows/pypy2.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI PyPy 2 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [pypy2] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 388d81dcbdf58e41088c368f69e08d81718c20a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:32:27 +0100 Subject: [PATCH 1254/2779] added pypy3.yaml --- .github/workflows/pypy3.yaml | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/pypy3.yaml diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml new file mode 100644 index 000000000..d1e8bf732 --- /dev/null +++ b/.github/workflows/pypy3.yaml @@ -0,0 +1,53 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI PyPy 3 + +#env: +# DEBUG: 1 + +on: + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + python-version: [pypy3] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-devops-python-tools + - name: init + run: make init + - name: build + run: make ci + - name: test + run: make test From 5ed7efb85ca23e9b53cf6519376630f1460ee789 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 15:34:07 +0100 Subject: [PATCH 1255/2779] updated README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f1f3c9027..7f479e65d 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Hari Sekhon - DevOps Python Tools [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) [![CI Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac%22) +[![CI Mac 10.15](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Mac%2010.15/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Mac+10.15%22) [![CI Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu%22) [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) @@ -69,7 +70,12 @@ Hari Sekhon - DevOps Python Tools [![CI Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine%22) [![CI Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Alpine+3%22) [![CI Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+2.7%22) +[![CI Python 3.5](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.5/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.5%22) [![CI Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.6%22) +[![CI Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.7%22) +[![CI Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Python+3.8%22) +[![CI PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+PyPy+2%22) +[![CI PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+PyPy+3%22) ### AWS, Docker, Spark / PySpark, Hadoop, HBase, Hive, Impala, Pig, Ambari, IPython and Linux Tools ### From d4e01a434b09008d7dc7332dc1a876fc4fd274e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 17:35:00 +0100 Subject: [PATCH 1256/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9731e35b3..0ff08f425 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9731e35b3bfeaece3b9706e3b2ec6f232041e22b +Subproject commit 0ff08f4255cc8b1534ef552b0f618ebff77d4477 From 1cb07cc1af7d65671e9dbfdabfb241bdb1a932c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 17:35:00 +0100 Subject: [PATCH 1257/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9d38d37af..86f6a3304 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9d38d37afeb88a09095cd04f5abd3d9e1ae8230b +Subproject commit 86f6a33046412ced2bf905082f29415a0f13a92b From b115e12862b73a94cdd45d826dafcb075ad742c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 17:59:23 +0100 Subject: [PATCH 1258/2779] updated find_active_server.py --- find_active_server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/find_active_server.py b/find_active_server.py index f759527eb..1a127cd54 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -120,11 +120,12 @@ from random import shuffle # Python 2 Queue vs Python 3 queue module :-/ if sys.version[0] == '2': - import Queue as queue + import Queue as queue # pylint: disable=import-error else: - import queue as queue + import queue as queue # pylint: disable=import-error try: - import requests + # false positive from pylint, queue is imported first + import requests # pylint: disable=wrong-import-order except ImportError: print(traceback.format_exc(), end='') sys.exit(4) From 92c9d920a4008f8fa05635c6f52a77786d6e08e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 19:42:26 +0100 Subject: [PATCH 1259/2779] updated headtail.py --- headtail.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/headtail.py b/headtail.py index ff061445a..ce9a43e43 100755 --- a/headtail.py +++ b/headtail.py @@ -41,7 +41,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.1' +__version__ = '0.3.2' class HeadTail(CLI): @@ -60,6 +60,7 @@ def __init__(self): self.sep = '-' * 80 self.docsep = '=' * 80 self.quiet = False + self.timeout_default = None def add_options(self): #self.timeout_default = 300 From 78e2932cd8682ae016c9160e45f25c99128dd340 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Apr 2020 23:02:22 +0100 Subject: [PATCH 1260/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 86f6a3304..0a5f65eae 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 86f6a33046412ced2bf905082f29415a0f13a92b +Subproject commit 0a5f65eae070fb0111c38e340d6dadf1e04c8a7f From 83f4ae1be5c6af89b1577f3b321c0d801bc5fa57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:46:34 +0100 Subject: [PATCH 1261/2779] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7f479e65d..32f290d82 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Hari Sekhon - DevOps Python Tools [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) +[![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) @@ -59,6 +60,7 @@ Hari Sekhon - DevOps Python Tools [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) [![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) +[![CI Ubuntu GitHub](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%20GitHub/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+GitHub%22) [![CI Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian%22) [![CI Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+8%22) [![CI Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+9%22) From 761b0f6344d4be2e8cd617d936a589b0741398cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:46:40 +0100 Subject: [PATCH 1262/2779] updated buddy.yml --- buddy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/buddy.yml b/buddy.yml index 9b897a700..19a4c04ce 100644 --- a/buddy.yml +++ b/buddy.yml @@ -28,11 +28,11 @@ docker_image_name: "library/ubuntu" docker_image_tag: "18.04" execute_commands: - - "apt update &&" - - "apt install -qy make &&" - - "make init &&" - - "make ci test" + - "apt update &&" + - "apt install -qy git make &&" + - "make init &&" + - "make ci test" volume_mappings: - - "/:/buddy/devops-python-tools" + - "/:/buddy/devops-python-tools" shell: "BASH" trigger_condition: "ALWAYS" From 7f92d4a64d43fa99e2487ac01f90b6980d3f6f4f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:47:04 +0100 Subject: [PATCH 1263/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0ff08f425..09e5ac956 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0ff08f4255cc8b1534ef552b0f618ebff77d4477 +Subproject commit 09e5ac956e613a4968fb50c2317f37857bd9e4f8 From ed84dacf4735a4a06eef7f92898f09e65231764d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:47:04 +0100 Subject: [PATCH 1264/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0a5f65eae..ea78f8b3d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0a5f65eae070fb0111c38e340d6dadf1e04c8a7f +Subproject commit ea78f8b3d7d4aaf5debfa92a59c2a281ae4b561a From 8a2fd8fb27ca4ac1dfcdcab617676c326223a4f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:47:28 +0100 Subject: [PATCH 1265/2779] renamed ubuntu.yaml to ubuntu_github.yaml --- .github/workflows/{ubuntu.yaml => ubuntu_github.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu.yaml => ubuntu_github.yaml} (100%) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu_github.yaml similarity index 100% rename from .github/workflows/ubuntu.yaml rename to .github/workflows/ubuntu_github.yaml From e4491bd6239b5ff9f453952fdb3828465a448fb8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:47:39 +0100 Subject: [PATCH 1266/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index c96960027..bd0f01189 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -11,7 +11,7 @@ # https://www.linkedin.com/in/harisekhon # -name: CI Ubuntu +name: CI Ubuntu GitHub #env: # DEBUG: 1 From 2a91910942868a1b3b412e58e61cf27d39458e8c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 21:48:00 +0100 Subject: [PATCH 1267/2779] added ubuntu.yaml --- .github/workflows/ubuntu.yaml | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/ubuntu.yaml diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml new file mode 100644 index 000000000..9e3506d6f --- /dev/null +++ b/.github/workflows/ubuntu.yaml @@ -0,0 +1,47 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ubuntu-latest + container: ubuntu:latest + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: apt-get update -qq && apt-get install -qy git make + - name: git clone + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive + - name: build + run: cd "/tmp/$repo" && make ci + - name: test + run: cd "/tmp/$repo" && make test From fdf294510f5f00352391185b2230c922a89f0db7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 22:44:53 +0100 Subject: [PATCH 1268/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 09e5ac956..ba415985e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 09e5ac956e613a4968fb50c2317f37857bd9e4f8 +Subproject commit ba415985e088004b4617353c52a4246bf1e81d7b From 32c2a168241debafda7804b3bed8f9abb26b646b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 11 Apr 2020 22:44:53 +0100 Subject: [PATCH 1269/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ea78f8b3d..401aa8728 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ea78f8b3d7d4aaf5debfa92a59c2a281ae4b561a +Subproject commit 401aa872810372013ee48435920c5c9f10c04220 From 1c49f975c1af79b9c65941b3f0ca79246b01b930 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 12 Apr 2020 21:20:57 +0100 Subject: [PATCH 1270/2779] added semaphore.yml --- .semaphore/semaphore.yml | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .semaphore/semaphore.yml diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml new file mode 100644 index 000000000..fe6a604ee --- /dev/null +++ b/.semaphore/semaphore.yml @@ -0,0 +1,71 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ + +version: v1.0 +name: devops-python-tools +agent: + machine: + type: e1-standard-2 + os_image: ubuntu1804 +execution_time_limit: + hours: 3 +blocks: + - name: Linux build + run: + when: "branch = 'master'" + #execution_time_limit: + # hours: 2 + task: + prologue: + commands: + - cache restore + jobs: + - name: install git & make + commands: + - sudo apt update -qq + - sudo apt install -qy git make + - name: build + commands: + - checkout + - make init + - make ci + - make test + epilogue: + commands: + - cache store + - name: Mac build + run: + when: "branch = 'masterX'" + task: + agent: + machine: + type: a1-standard-4 + os_image: macos-mojave-xcode10 + #os_image: macos-mojave-xcode11 + prologue: + commands: + - cache restore + jobs: + - name: build + commands: + - checkout + - make init + - make ci + - make test + epilogue: + commands: + - cache store From 8a6f7bd6af2b194c574217deb52a98367b807fb7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 12 Apr 2020 21:25:17 +0100 Subject: [PATCH 1271/2779] updated Makefile --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 4eb3ff5e1..7fa953349 100755 --- a/Makefile +++ b/Makefile @@ -62,7 +62,12 @@ build: init @# executing in sh where type is not available @#type -P python which python || : + which python2 || : + which python3 || : python -V || : + which pip || : + which pip2 || : + which pip3 || : pip -V || : if [ -z "$(CPANM)" ]; then make; exit $$?; fi From 501520a0308eb4ad322dd4ad310bc7795b3ec45b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 14 Apr 2020 13:38:54 +0100 Subject: [PATCH 1272/2779] updated Makefile --- Makefile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 7fa953349..e241aabb7 100755 --- a/Makefile +++ b/Makefile @@ -61,14 +61,12 @@ build: init @# executing in sh where type is not available @#type -P python - which python || : - which python2 || : - which python3 || : - python -V || : - which pip || : - which pip2 || : - which pip3 || : - pip -V || : + which python && python -V || : + which python2 && python2 -V || : + which python3 && python3 -V || : + which pip && pip -V || : + which pip2 && pip2 -V || : + which pip3 && pip3 -V || : if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python From 32dd12db959e2e754c24f30ac0b505e3c68ea9d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 15 Apr 2020 15:44:52 +0100 Subject: [PATCH 1273/2779] updated Makefile --- Makefile | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index e241aabb7..3376063e7 100755 --- a/Makefile +++ b/Makefile @@ -53,39 +53,25 @@ ifndef SKIP_PARQUET endif .PHONY: build -build: init +build: init python-version @echo ========================= @echo DevOps Python Tools Build @echo ========================= @$(MAKE) git-summary - @# executing in sh where type is not available - @#type -P python - which python && python -V || : - which python2 && python2 -V || : - which python3 && python3 -V || : - which pip && pip -V || : - which pip2 && pip2 -V || : - which pip3 && pip3 -V || : - if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python if type apk 2>/dev/null; then $(MAKE) apk-packages-extra; fi if type apt-get 2>/dev/null; then $(MAKE) apt-packages-extra; fi - $(MAKE) python - # executing in sh where type is not available - #type -P python - which python - python -V - pip -V + $(MAKE) python .PHONY: init init: git submodule update --init --recursive .PHONY: python -python: +python: python-version cd pylib && $(MAKE) @# don't pull parquet tools in to docker image by default, will bloat it @# can fetch separately by running 'make parquet-tools' if you really want to From 3be70ea4f98ebce70c168432c0ac127fd1149b9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 14:58:32 +0100 Subject: [PATCH 1274/2779] updated .semaphore --- .semaphore/semaphore.yml | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fe6a604ee..26ac24a3d 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -30,17 +30,20 @@ blocks: #execution_time_limit: # hours: 2 task: + env_vars: + # to match /usr/local/bin/pip version from $PATH + - name: PYTHON + value: python3.8 prologue: commands: - cache restore + # each job is separate and could be run on a separate machine so all steps must be together jobs: - - name: install git & make - commands: - - sudo apt update -qq - - sudo apt install -qy git make - name: build commands: - checkout + - sudo apt update -qq + - sudo apt install -qy git make - make init - make ci - make test @@ -49,8 +52,17 @@ blocks: - cache store - name: Mac build run: - when: "branch = 'masterX'" + when: "branch = 'master'" task: + # because otherwise on Mac it uses /usr/bin/python (2.7) but /usr/local/bin/pip (python 3.8) + env_vars: + # to match /usr/local/bin/pip version from $PATH + - name: PYTHON + value: python3 + # must be quoted to force string, otherwise pipeline fails to run with this parsing error: + # Error: [{"Type mismatch. Expected String but got Integer.", "#/blocks/1/task/env_vars/1/value"}] + #- name: DEBUG + # value: "1" agent: machine: type: a1-standard-4 @@ -59,6 +71,23 @@ blocks: prologue: commands: - cache restore + # fix for: + # pip is configured with locations that require TLS/SSL, however the ssl module in Python is not available. + - brew install openssl + - brew reinstall python + - brew reinstall wget + # avoid Mac SSL errors: + # + # ERROR: Loading command: install (LoadError) + # dlopen(/Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle, 9): Library not loaded: /usr/local/opt/openssl/lib/libssl.1.0.0.dylib + # Referenced from: /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle + # Reason: image not found - /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle + # ERROR: While executing gem ... (NoMethodError) + # undefined method `invoke_with_build_args' for nil:NilClass# + # + - rbenv global system + # also considered this: + # - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done jobs: - name: build commands: From 289b48b51761aa3fb56c8f241c207a83a7484dbb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 14:59:34 +0100 Subject: [PATCH 1275/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ba415985e..e9323e43b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ba415985e088004b4617353c52a4246bf1e81d7b +Subproject commit e9323e43bc5e8f39d821be5fbca3522a84410536 From a2a316ebebbfe10904191bdc89f4bdc165a2719e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 14:59:34 +0100 Subject: [PATCH 1276/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 401aa8728..a1f3c1116 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 401aa872810372013ee48435920c5c9f10c04220 +Subproject commit a1f3c1116c613e32e7a85df01bf819b4ecc16dbc From 565c7d92d0fff4ad112018025af87f8a328abbee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:10:56 +0100 Subject: [PATCH 1277/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a1f3c1116..9e5deb8c3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a1f3c1116c613e32e7a85df01bf819b4ecc16dbc +Subproject commit 9e5deb8c32579f1e1ce7e11475a49e8bd9e33d4b From eec13571c5fd2078a6d6eb6bcf4fb746bddc5e22 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:11:19 +0100 Subject: [PATCH 1278/2779] updated pypy2.yaml pypy3.yaml --- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 28a8a6445..c71f2f9e5 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -32,7 +32,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] python-version: [pypy2] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index d1e8bf732..ad306147d 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -32,7 +32,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - #python-version: [2.7, 3.6, 3.7, 3.8, pypy2, pypy3] + #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] python-version: [pypy3] steps: - uses: actions/checkout@v2 From 9baeeafdbb177819cc7a2fc81df8169dd0029b9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:12:38 +0100 Subject: [PATCH 1279/2779] updated Makefile --- Makefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Makefile b/Makefile index 3376063e7..a6706c6ab 100755 --- a/Makefile +++ b/Makefile @@ -185,11 +185,6 @@ test: test-lib basic-test: test-lib bash-tools/check_all.sh -.PHONY: test2 -test2: - cd pylib && $(MAKE) test2 - tests/all.sh - .PHONY: install install: build @echo "No installation needed, just add '$(PWD)' to your \$$PATH" From 4da2fa60f66774497ea031e468a4af2ee041954f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:30:53 +0100 Subject: [PATCH 1280/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9e5deb8c3..865dd9fe6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9e5deb8c32579f1e1ce7e11475a49e8bd9e33d4b +Subproject commit 865dd9fe6296f5611115adc2ba11752a3e11f80b From 6aa4b5cd71872b09f7058018c90c43e3bc561811 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:30:56 +0100 Subject: [PATCH 1281/2779] updated .travis.yml --- .travis.yml | 63 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0ef2d390c..8cd9b2f40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,24 +20,61 @@ version: ~> 1.0 # - linux # - osx +python: + - "2.7" + #- "3.4" supported by pip as of March 2019 + - "3.5" + - "3.6" + - "3.7" + - "3.8" + - "pypy" # currently Python 2.7.13, PyPy 7.1.1 + - "pypy3" # currently Python 3.6.1, PyPy 7.1.1-beta0 + matrix: + fast_finish: true include: + # numpy has gone 2.7+ only now, so had to drop Python 2.6 support - os: linux language: python - python: - # - "2.6" - - "2.7" - # MySQL in lib doesn't build from pip in Python 3 - # - "3.2" - # - "3.3" - # - "3.4" - # - "3.5" - # python-krbV fails to compile on PyPy - # - "pypy" - # - "pypy3" - # workaround is to use generic and install to system python + python: "2.7" + - os: osx - language: generic + language: generic # workaround since Mac doesn't have Python support yet, so install to system Python + + - os: linux + language: python + python: "3.5" + + - os: linux + language: python + python: "3.6" + + - os: linux + language: python + python: "3.7" + + - os: linux + language: python + python: "3.8" + + # python-krbV fails to compile on PyPy + - os: linux + language: python + python: "pypy" + + - os: linux + language: python + python: "pypy3" + + +# allow_failures: +# - python: "3.4" +# - python: "3.5" +# - python: "3.6" +# - python: "3.7" +# - python: "3.8" +# - python: "pypy" +# - python: "pypy3" dist: trusty From 792243c5c1d82c3ca4419477424b9f323ac92255 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 17:32:13 +0100 Subject: [PATCH 1282/2779] updated .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8cd9b2f40..7ca66678f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ matrix: # - python: "pypy" # - python: "pypy3" -dist: trusty +#dist: trusty sudo: required From 032ecb2ffd12585131b113d6b0f06e7d1d489846 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 18:59:42 +0100 Subject: [PATCH 1283/2779] updated .travis.yml --- .travis.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7ca66678f..cf0870f49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,10 +57,15 @@ matrix: language: python python: "3.8" - # python-krbV fails to compile on PyPy - - os: linux - language: python - python: "pypy" + # python-krbV fails to compile on PyPy + # + # psutil doesn't build: + # + # RuntimeError: broken / incompatible Python implementation, see: https://github.com/giampaolo/psutil/issues/1659 + # + #- os: linux + # language: python + # python: "pypy" - os: linux language: python From 2774c67202b883d59dd2d942aa458b037616f703 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 19:15:59 +0100 Subject: [PATCH 1284/2779] updated buddy.yml --- buddy.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/buddy.yml b/buddy.yml index 19a4c04ce..8257e4b8d 100644 --- a/buddy.yml +++ b/buddy.yml @@ -27,11 +27,12 @@ working_directory: "/buddy/devops-python-tools" docker_image_name: "library/ubuntu" docker_image_tag: "18.04" + setup_commands: + - apt update + - apt install -qy git make execute_commands: - - "apt update &&" - - "apt install -qy git make &&" - - "make init &&" - - "make ci test" + - make init + - make ci test volume_mappings: - "/:/buddy/devops-python-tools" shell: "BASH" From 0a5cbab7d19a2875b20e3bcba1563f694bbfb7c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 17 Apr 2020 19:16:44 +0100 Subject: [PATCH 1285/2779] updated .appveyor.yml --- .appveyor.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 7934f0253..65331b992 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -29,6 +29,15 @@ environment: on_finish: # set this in Settings -> Environment dynamically instead of here #- sh: export APPVEYOR_SSH_BLOCK=true + # + # workaround for https://github.com/appveyor/ci/issues/3373 + # and https://github.com/appveyor/ci/issues/3384 + # + # has since been added to AppVeyor's own scripts: + # + # https://github.com/appveyor/ci/pull/3385 + # + #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/install_openssh.sh' | bash -e - - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - install: From 9aa2a1113a4cceefe02f543cf1fcbfa71c6fbba7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Apr 2020 14:01:36 +0100 Subject: [PATCH 1286/2779] updated Makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a6706c6ab..be45f98a7 100755 --- a/Makefile +++ b/Makefile @@ -21,15 +21,15 @@ # # Alpine: # -# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make # # Debian / Ubuntu: # -# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make # # RHEL / CentOS: # -# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && make +# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make # =================== From 61f627320ebea997558c603f4393f914fd86cd53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 24 Apr 2020 14:18:09 +0100 Subject: [PATCH 1287/2779] updated welcome.py --- welcome.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/welcome.py b/welcome.py index bde562ac9..60d6533af 100755 --- a/welcome.py +++ b/welcome.py @@ -45,7 +45,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '2.0.3' +__version__ = '2.0.4' class Welcome(CLI): @@ -59,7 +59,7 @@ def __init__(self): self.timeout_default = 20 @staticmethod - def case_user(user): + def titlecase_user(user): if user == 'root': user = user.upper() elif len(user) < 4 or re.search(r'\d', user): @@ -76,7 +76,7 @@ def construct_msg(self): # print("invalid user '%s' determined from environment variable $USER, failed regex validation" % user) print("invalid user '%s' returned by getpass.getuser(), failed regex validation" % user) sys.exit(ERRORS['CRITICAL']) - user = self.case_user(user) + user = self.titlecase_user(user) msg = 'Welcome %s - ' % user last = '' if which("last"): From 84cc517484fccc3aca372c41364bac9a7470fea7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:34:36 +0100 Subject: [PATCH 1288/2779] updated .appveyor.yml --- .appveyor.yml | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index e367a94c7..65331b992 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -17,7 +17,48 @@ image: Ubuntu +# https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ +environment: + APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension + +# enable SSH session accessible via my public key +#init: +# - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + +# more useful at end to leverage .appveyor.yml tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to do all that manually in SSH session +on_finish: + # set this in Settings -> Environment dynamically instead of here + #- sh: export APPVEYOR_SSH_BLOCK=true + # + # workaround for https://github.com/appveyor/ci/issues/3373 + # and https://github.com/appveyor/ci/issues/3384 + # + # has since been added to AppVeyor's own scripts: + # + # https://github.com/appveyor/ci/pull/3385 + # + #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/install_openssh.sh' | bash -e - + - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + install: +# workaround for: +# Some packages could not be installed. This may mean that you have +# requested an impossible situation or if you are using the unstable +# distribution that some required packages have not yet been created +# or been moved out of Incoming. +# The following information may help to resolve the situation: +# +# The following packages have unmet dependencies: +# mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed +# E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. +# devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed +# make[2]: *** [apt-packages] Error 123 +# make[2]: Leaving directory '/home/appveyor/projects/pylib' +# devops-python-tools/Makefile.in:212: recipe for target 'system-packages' failed +# +# adding "|| :" to the end of these commands causes them to be silently ignored! +- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list +- sudo apt purge -qy --allow-change-held-packages mssql-server - sudo apt update -qq - sudo apt install -qy git make - make From c76907a8c59c9cb8691f4082116bc5cbce40db3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:35:04 +0100 Subject: [PATCH 1289/2779] updated config.yml --- .circleci/config.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7217d95a7..114368b87 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,10 +19,20 @@ version: 2.1 jobs: build: + # technically a docker image is a better choice than machine + # but we want to introduce some native environment variation + # between build systems in order to test the repo's build automation is robust machine: - #image: ubuntu-1604:201903-01 image: default + #image: ubuntu-1604:201903-01 + # set to an actual docker image when running locally using circle_ci_job.sh + # docker image must have git installed to do the checkout + # so using harisekhon/dev:ubuntu instead of base ubuntu image + #image: harisekhon/dev:ubuntu steps: + # to allow docker networking to work + - run: sudo sysctl net.ipv4.ip_forward=1 + - run: sudo service docker restart - checkout - run: make init - run: make From 36392354a84e097e04f65fd794dc993e4dbad684 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:50:22 +0100 Subject: [PATCH 1290/2779] updated buddy.yml --- buddy.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/buddy.yml b/buddy.yml index 19a4c04ce..8257e4b8d 100644 --- a/buddy.yml +++ b/buddy.yml @@ -27,11 +27,12 @@ working_directory: "/buddy/devops-python-tools" docker_image_name: "library/ubuntu" docker_image_tag: "18.04" + setup_commands: + - apt update + - apt install -qy git make execute_commands: - - "apt update &&" - - "apt install -qy git make &&" - - "make init &&" - - "make ci test" + - make init + - make ci test volume_mappings: - "/:/buddy/devops-python-tools" shell: "BASH" From ada7e496c047903b272b3b918b11b7cca16d39d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:50:28 +0100 Subject: [PATCH 1291/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 26ac24a3d..f622c07a1 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -66,8 +66,7 @@ blocks: agent: machine: type: a1-standard-4 - os_image: macos-mojave-xcode10 - #os_image: macos-mojave-xcode11 + os_image: macos-xcode11 prologue: commands: - cache restore From 125fbc61716c7201dc0b518e14ead22fa2b7ea00 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:51:12 +0100 Subject: [PATCH 1292/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e9323e43b..bf60f35d3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e9323e43bc5e8f39d821be5fbca3522a84410536 +Subproject commit bf60f35d33bf2aa38f18e7fb09d6f9a2125a7400 From 8084bfaa3b6d81f19be42ca0def6080b37ca2b4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 15:51:12 +0100 Subject: [PATCH 1293/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 865dd9fe6..ae53903b5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 865dd9fe6296f5611115adc2ba11752a3e11f80b +Subproject commit ae53903b5f8a998084207b746d4fe4c956ef57b4 From 271af24995e6bfdeafc6f7ae565c570407f79d47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 17:58:55 +0100 Subject: [PATCH 1294/2779] added ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/ubuntu_20.04.yaml diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml new file mode 100644 index 000000000..5f38e935c --- /dev/null +++ b/.github/workflows/ubuntu_20.04.yaml @@ -0,0 +1,47 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/devops-python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: CI Ubuntu 20.04 + +#env: +# DEBUG: 1 + +on: # [push] + push: + branches: + - master + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 7 * * *' + +jobs: + build: + #name: build + timeout-minutes: 60 + runs-on: ubuntu-latest + container: ubuntu:20.04 + env: + repo: devops-python-tools + steps: + # untars repo in docker container so git submodule update fails + #- uses: actions/checkout@v2 + - name: install git & make + run: apt-get update -qq && apt-get install -qy git make + - name: git clone + run: cd /tmp && git clone "https://github.com/harisekhon/$repo" + - name: init + run: cd "/tmp/$repo" && git submodule update --init --recursive + - name: build + run: cd "/tmp/$repo" && make ci + - name: test + run: cd "/tmp/$repo" && make test From 7879f313019aef4e2c7b15082ee142615fce2442 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 19:54:53 +0100 Subject: [PATCH 1295/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 32f290d82..f5893fc5f 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Hari Sekhon - DevOps Python Tools [![CI Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+14.04%22) [![CI Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+16.04%22) [![CI Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+18.04%22) +[![CI Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+20.04%22) [![CI Ubuntu GitHub](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Ubuntu%20GitHub/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Ubuntu+GitHub%22) [![CI Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian%22) [![CI Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CI%20Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CI+Debian+8%22) From b364e74c6496b3265857382cfc350cec67e7816c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 20:16:57 +0100 Subject: [PATCH 1296/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bf60f35d3..90120262f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bf60f35d33bf2aa38f18e7fb09d6f9a2125a7400 +Subproject commit 90120262ff16a7c25a046d43f823335c5a442ded From d0d546213f1fc3d68098a72f92afae40202f1e4d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 20:16:57 +0100 Subject: [PATCH 1297/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ae53903b5..918826210 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ae53903b5f8a998084207b746d4fe4c956ef57b4 +Subproject commit 918826210859758647ab6dc02d8a6af069e6e760 From b52989e0273123646add41b42d8da97e731e16cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 20:37:15 +0100 Subject: [PATCH 1298/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 90120262f..81fa2ce69 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 90120262ff16a7c25a046d43f823335c5a442ded +Subproject commit 81fa2ce6926642ebd671e74b3bc0b4a35c4208f4 From a64154e26a947753c438037c00a958c957a41589 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 29 Apr 2020 20:37:15 +0100 Subject: [PATCH 1299/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 918826210..d866efd80 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 918826210859758647ab6dc02d8a6af069e6e760 +Subproject commit d866efd806a74b97b687902796d26f688be87a54 From 740399c7be3652682429b687f324ccc90df893b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 18:45:31 +0100 Subject: [PATCH 1300/2779] updated Makefile --- Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index be45f98a7..f13ff903a 100755 --- a/Makefile +++ b/Makefile @@ -53,11 +53,14 @@ ifndef SKIP_PARQUET endif .PHONY: build -build: init python-version +build: init @echo ========================= @echo DevOps Python Tools Build @echo ========================= @$(MAKE) git-summary + # defer via external sub-call, otherwise will result in error like + # make: *** No rule to make target 'python-version', needed by 'build'. Stop. + $(MAKE) python-version if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python @@ -71,7 +74,10 @@ init: git submodule update --init --recursive .PHONY: python -python: python-version +python: + # defer via external sub-call, otherwise will result in error like + # make: *** No rule to make target 'python-version', needed by 'build'. Stop. + $(MAKE) python-version cd pylib && $(MAKE) @# don't pull parquet tools in to docker image by default, will bloat it @# can fetch separately by running 'make parquet-tools' if you really want to From 45e2ef423b28e38586233adaf87ef660c2b7b946 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 18:46:44 +0100 Subject: [PATCH 1301/2779] updated Makefile --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f13ff903a..d939369eb 100755 --- a/Makefile +++ b/Makefile @@ -58,9 +58,10 @@ build: init @echo DevOps Python Tools Build @echo ========================= @$(MAKE) git-summary + @echo # defer via external sub-call, otherwise will result in error like # make: *** No rule to make target 'python-version', needed by 'build'. Stop. - $(MAKE) python-version + @$(MAKE) python-version if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python @@ -77,7 +78,7 @@ init: python: # defer via external sub-call, otherwise will result in error like # make: *** No rule to make target 'python-version', needed by 'build'. Stop. - $(MAKE) python-version + @$(MAKE) python-version cd pylib && $(MAKE) @# don't pull parquet tools in to docker image by default, will bloat it @# can fetch separately by running 'make parquet-tools' if you really want to From a6426138b4c8f4c864359afe02d1037363894f0b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 18:46:48 +0100 Subject: [PATCH 1302/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 81fa2ce69..fa89f29a2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 81fa2ce6926642ebd671e74b3bc0b4a35c4208f4 +Subproject commit fa89f29a29316ccfbd4fe33fc9207d4a9ea507dd From 76ab0836a999fa5f885e1d99f9437413d04e6e0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 18:46:49 +0100 Subject: [PATCH 1303/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d866efd80..7504ea948 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d866efd806a74b97b687902796d26f688be87a54 +Subproject commit 7504ea948018ad50374e1d0cae08d4baa44316cd From 9b704e57903b589d485139e5963852e80aa565f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 30 Apr 2020 19:37:16 +0100 Subject: [PATCH 1304/2779] updated .appveyor.yml --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 65331b992..16c3722e5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -38,7 +38,7 @@ on_finish: # https://github.com/appveyor/ci/pull/3385 # #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/install_openssh.sh' | bash -e - - - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi install: # workaround for: From fb55c47d7f7b7cc112911b6f465f6784f2b92f94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 May 2020 14:57:14 +0100 Subject: [PATCH 1305/2779] updated .appveyor.yml --- .appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 16c3722e5..7540e70a9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -59,6 +59,8 @@ install: # adding "|| :" to the end of these commands causes them to be silently ignored! - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list - sudo apt purge -qy --allow-change-held-packages mssql-server +# this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 +#- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages - sudo apt update -qq - sudo apt install -qy git make - make From 9488e1d593cdc1b2ca29693a72150411382c2c96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 15 May 2020 14:57:24 +0100 Subject: [PATCH 1306/2779] updated .semaphore --- .semaphore/semaphore.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index f622c07a1..4cfd2cb8e 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -31,9 +31,11 @@ blocks: # hours: 2 task: env_vars: - # to match /usr/local/bin/pip version from $PATH + # $PATH selects /usr/bin/python and /usr/local/bin/pip which are mismatched versions of Python - name: PYTHON - value: python3.8 + value: python3 + - name: PIP + value: pip3 prologue: commands: - cache restore From 914ac76360a2f70e4c2872b3b9c023a7c15b5274 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 May 2020 17:27:42 +0100 Subject: [PATCH 1307/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fa89f29a2..e9371b29e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fa89f29a29316ccfbd4fe33fc9207d4a9ea507dd +Subproject commit e9371b29e17e4ff8a091fbd1ce589c53ebea3679 From eddc03110a7d5f0ccf5dbd0da957703d97526ad3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 16 May 2020 17:27:42 +0100 Subject: [PATCH 1308/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7504ea948..545134be7 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7504ea948018ad50374e1d0cae08d4baa44316cd +Subproject commit 545134be7d446b2e67b8cce0deba69ca0bbba2bd From 238ccca41414021e28b9b45f6e828744d283f2e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:08:06 +0100 Subject: [PATCH 1309/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f5893fc5f..0f30f8aa9 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Hari Sekhon - DevOps Python Tools [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) -[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/project/byKey/b40735fb89e7d989dbaf5659a9af9a20) +[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) From 996cdf24b1258ff1e1a09576003f599468e5deaf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:26:29 +0100 Subject: [PATCH 1310/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e9371b29e..f7318108b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e9371b29e17e4ff8a091fbd1ce589c53ebea3679 +Subproject commit f7318108b1ebfe6e013eed1d4266e1f5da46ff7a From 1c21f25a6adc62d3426049f9eaf9143d86e23161 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:26:29 +0100 Subject: [PATCH 1311/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 545134be7..40fd56a17 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 545134be7d446b2e67b8cce0deba69ca0bbba2bd +Subproject commit 40fd56a17e57b5ee616cb51a552ffe7b9f868aa4 From 77ac613cf4b2160c147a1846f30a26ff0390bae1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:46:14 +0100 Subject: [PATCH 1312/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f7318108b..c2f5c76bc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f7318108b1ebfe6e013eed1d4266e1f5da46ff7a +Subproject commit c2f5c76bca9aa6e4b6aba9398fbf0e861ed5bceb From f1fff3d5835d6e816fecf962bfb3b61f1a9baa43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 19 May 2020 18:46:14 +0100 Subject: [PATCH 1313/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 40fd56a17..f6719de55 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 40fd56a17e57b5ee616cb51a552ffe7b9f868aa4 +Subproject commit f6719de55889670fc458230b6b20a1f47bb6ecea From b0be3b6cc99beeccbe49d49aac4200dd23cc09fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 23 May 2020 17:23:35 +0100 Subject: [PATCH 1314/2779] updated README.md --- README.md | 67 +++++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 0f30f8aa9..95bea4b8f 100644 --- a/README.md +++ b/README.md @@ -158,19 +158,23 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days -- [Hadoop](http://hadoop.apache.org/) & NoSQL: - - [Spark](https://spark.apache.org/) & Data Format Converters: - - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter - - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas - - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```xml_to_json.py``` - XML to JSON converter - - ```json_to_xml.py``` - JSON to XML converter - - ```json_to_yaml.py``` - JSON to YAML converter - - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - - see also ```validate_*.py``` further down for all these formats and more +- [Docker](https://www.docker.com/): + - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` + - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (official `docker search` is limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` + - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` +- [Spark](https://spark.apache.org/) & Data Format Converters: + - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter + - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter + - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas + - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas + - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter + - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter + - ```xml_to_json.py``` - XML to JSON converter + - ```json_to_xml.py``` - JSON to XML converter + - ```json_to_yaml.py``` - JSON to YAML converter + - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - see also ```validate_*.py``` further down for all these formats and more +- [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: - [Ambari](https://hortonworks.com/apache/ambari/): - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API - list blueprints @@ -220,27 +224,22 @@ Environment variables are supported for convenience and also to hide credentials - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop - - ```ipython-notebook-pyspark.py``` - per-user authenticated IPython Notebook + PySpark integration to allow each user to auto-create their own password protected IPython Notebook running Spark - - ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required - - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) -- [Docker](https://www.docker.com/): - - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (official `docker search` is limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` +- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) + - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required + - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA + - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA + - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA + - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) + - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node + - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node + - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator + - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server + - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server + - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node + - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node + - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI From a29eaf12a7946e9b18d3ab13816821e730114d79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 23 May 2020 17:25:29 +0100 Subject: [PATCH 1315/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 95bea4b8f..289b61976 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ Environment variables are supported for convenience and also to hide credentials - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch). See further down for more details and sub-programs that simplify usage for the most common cluster technologies - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [AWS](https://aws.amazon.com/): - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) From 17edfb63b0b3db4761fe083342ab070f55f28632 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 23 May 2020 17:31:08 +0100 Subject: [PATCH 1316/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 289b61976..305b8b1d6 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Environment variables are supported for convenience and also to hide credentials - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch). See further down for more details and sub-programs that simplify usage for the most common cluster technologies + - ```find_active_server.py``` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [AWS](https://aws.amazon.com/): - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) From 84997dbc83f1a17a61225c3a762fd485069734a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 24 May 2020 19:23:51 +0100 Subject: [PATCH 1317/2779] set regex matching to apply only to file basename and not full path --- find_duplicate_files.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 00b45fe2e..7556c290c 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -33,7 +33,7 @@ 4. regex capture matching portion - specify a regex to match against the filenames with capture (brackets) and the captured portion will be compared among files. If no capture brackets are detected then will treat the entire regex as the capture. - Regex is case insensitive by default + Regex is case insensitive by default and applies only to the file's basename Can restrict methods of finding duplicates to any combination of --name / --size / --checksum (checksum implies size as an efficiency shortcut) / --regex. If none are specified then will try name, size + checksum. If specifying any one of @@ -81,7 +81,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.4' +__version__ = '0.6.0' class FindDuplicateFiles(CLI): @@ -165,7 +165,7 @@ def process_args(self): log_option('compare by name', self.compare_by_name) log_option('compare by size', self.compare_by_size) log_option('compare by checksum', self.compare_by_checksum) - log_option('compare by regex', True if self.regex else False) + log_option('compare by regex', bool(self.regex)) return args @staticmethod @@ -297,6 +297,7 @@ def check_path(self, path): def is_file_dup(self, filepath): log.debug("checking file path '%s'", filepath) + # pylint: disable=no-else-return if os.path.islink(filepath): log.debug("ignoring symlink '%s'", filepath) return False @@ -340,8 +341,7 @@ def is_file_dup_by_name(self, filepath): self.dups_by_name[basename].add(self.files[basename]) self.dups_by_name[basename].add(filepath) return True - else: - self.files[basename] = filepath + self.files[basename] = filepath return False def is_file_dup_by_size(self, filepath): @@ -397,7 +397,9 @@ def is_file_dup_by_hash(self, filepath): return False def is_file_dup_by_regex(self, filepath): - match = re.search(self.regex, filepath) + #match = re.search(self.regex, filepath) + basename = os.path.basename(filepath) + match = re.search(self.regex, basename) if match: log.debug("regex matched file '%s'", filepath) if match.groups(): @@ -407,8 +409,7 @@ def is_file_dup_by_regex(self, filepath): self.dups_by_regex[capture].add(self.regex_captures[capture]) self.dups_by_regex[capture].add(filepath) return True - else: - self.regex_captures[capture] = filepath + self.regex_captures[capture] = filepath else: log.error('no capture detected! Did you forget to specify the (brackets) to capture in the regex?') return False From 122b0fea4309b4e5937fbee157e464b8a0227ed7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 25 May 2020 09:57:24 +0100 Subject: [PATCH 1318/2779] defaulted to capturing entire regex when no capture group is given --- find_duplicate_files.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 7556c290c..b1f35742e 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -81,7 +81,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.0' +__version__ = '0.6.1' class FindDuplicateFiles(CLI): @@ -404,14 +404,14 @@ def is_file_dup_by_regex(self, filepath): log.debug("regex matched file '%s'", filepath) if match.groups(): capture = match.group(1) - if capture in self.regex_captures: - self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set()) - self.dups_by_regex[capture].add(self.regex_captures[capture]) - self.dups_by_regex[capture].add(filepath) - return True - self.regex_captures[capture] = filepath else: - log.error('no capture detected! Did you forget to specify the (brackets) to capture in the regex?') + capture = match.group(0) + if capture in self.regex_captures: + self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set()) + self.dups_by_regex[capture].add(self.regex_captures[capture]) + self.dups_by_regex[capture].add(filepath) + return True + self.regex_captures[capture] = filepath return False From f1ccc9670bb9b6f3d566a7b8ddf445a7064356bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 May 2020 11:49:42 +0100 Subject: [PATCH 1319/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c2f5c76bc..3f3c2f918 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c2f5c76bca9aa6e4b6aba9398fbf0e861ed5bceb +Subproject commit 3f3c2f918b242e32a112a182ac461ab66331bab0 From f40eeceffd4a6e698ab6d71401be24eba507b9bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 26 May 2020 11:49:44 +0100 Subject: [PATCH 1320/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f6719de55..93c911150 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f6719de55889670fc458230b6b20a1f47bb6ecea +Subproject commit 93c9111508dbbb756f99c8062c095580cf6bc203 From d64e76f6ec5142087f5bf47ac5031be38b106dab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 14:28:57 +0100 Subject: [PATCH 1321/2779] added barclaycard_crunch_accounting_csv_statement_converter.sh --- ...unch_accounting_csv_statement_converter.sh | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100755 crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh new file mode 100755 index 000000000..5b6ed56f8 --- /dev/null +++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +#. "$srcdir/lib/utils.sh" + +# statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv +STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" + +converter="$srcdir/../crunch_accounting_csv_statement_converter.py" + +for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + latest_crunch_statement="$crunch_statement" + fi +done + +get_starting_balance(){ + local crunch_statement="$1" + if ! [[ "$crunch_statement" =~ _crunch.csv$ ]]; then + echo "invalid statement passed to get_starting_balance, must be *_crunch.csv" >&2 + exit 1 + fi + tail -n 1 "$crunch_statement" | awk -F, '{print $4}' +} + +if [ -n "${latest_crunch_statement:-}" ]; then + echo "latest crunch statement is $latest_crunch_statement" + starting_balance="$(get_starting_balance "$latest_crunch_statement")" +else + starting_balance="${STARTING_BALANCE:-}" + if [ -n "$starting_balance" ]; then + echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$LAST_BALANCE" >&2 + exit 1 + fi +fi + +echo "starting balance: $starting_balance" + +passed_latest_statement=0 + +for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + if [ "$crunch_statement" = "$latest_crunch_statement" ]; then + passed_latest_statement=1 + fi + continue + fi + if [ $passed_latest_statement -lt 1 ]; then + continue + fi + "$converter" --credit-card --reverse-order --starting-balance "$starting_balance" "$statement" + starting_balance="$(get_starting_balance "$crunch_statement")" +done From a79b43a99638617883d7cf59e0364e7f638a3945 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 14:39:55 +0100 Subject: [PATCH 1322/2779] updated barclaycard_crunch_accounting_csv_statement_converter.sh --- ...unch_accounting_csv_statement_converter.sh | 52 ++----------------- 1 file changed, 3 insertions(+), 49 deletions(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh index 5b6ed56f8..cbf3568e3 100755 --- a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh +++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh @@ -18,55 +18,9 @@ set -euo pipefail srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1090 -#. "$srcdir/lib/utils.sh" +. "$srcdir/lib.sh" # statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv -STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" +export STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" -converter="$srcdir/../crunch_accounting_csv_statement_converter.py" - -for statement in $STATEMENT_GLOB; do - crunch_statement="${statement%.csv}_crunch.csv" - if [ -f "$crunch_statement" ]; then - latest_crunch_statement="$crunch_statement" - fi -done - -get_starting_balance(){ - local crunch_statement="$1" - if ! [[ "$crunch_statement" =~ _crunch.csv$ ]]; then - echo "invalid statement passed to get_starting_balance, must be *_crunch.csv" >&2 - exit 1 - fi - tail -n 1 "$crunch_statement" | awk -F, '{print $4}' -} - -if [ -n "${latest_crunch_statement:-}" ]; then - echo "latest crunch statement is $latest_crunch_statement" - starting_balance="$(get_starting_balance "$latest_crunch_statement")" -else - starting_balance="${STARTING_BALANCE:-}" - if [ -n "$starting_balance" ]; then - echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$LAST_BALANCE" >&2 - exit 1 - fi -fi - -echo "starting balance: $starting_balance" - -passed_latest_statement=0 - -for statement in $STATEMENT_GLOB; do - crunch_statement="${statement%.csv}_crunch.csv" - if [ -f "$crunch_statement" ]; then - if [ "$crunch_statement" = "$latest_crunch_statement" ]; then - passed_latest_statement=1 - fi - continue - fi - if [ $passed_latest_statement -lt 1 ]; then - continue - fi - "$converter" --credit-card --reverse-order --starting-balance "$starting_balance" "$statement" - starting_balance="$(get_starting_balance "$crunch_statement")" -done +generate_crunch_statements From 26afc75e240ff1d126e82e34f65cf7ca72924aa4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 14:39:59 +0100 Subject: [PATCH 1323/2779] added lib.sh --- .../lib.sh | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 crunch_accounting_csv_statement_converter_scripts/lib.sh diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh new file mode 100755 index 000000000..f6079e7fb --- /dev/null +++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +#. "$srcdir/lib/utils.sh" + +converter="$srcdir/../crunch_accounting_csv_statement_converter.py" + +get_latest_crunch_statement(){ + local latest_crunch_statement + for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + latest_crunch_statement="$crunch_statement" + fi + done + if [ -n "$latest_crunch_statement" ]; then + echo "$latest_crunch_statement" + fi +} + +get_final_balance_from_statement(){ + local crunch_statement="$1" + if ! [[ "$crunch_statement" =~ _crunch.csv$ ]]; then + echo "invalid statement passed to get_final_balance_from_statement(), must be *_crunch.csv" >&2 + exit 1 + fi + tail -n 1 "$crunch_statement" | awk -F, '{print $4}' +} + +get_starting_balance(){ + local starting_balance + if [ -n "${latest_crunch_statement:-}" ]; then + echo "latest crunch statement is $latest_crunch_statement" >&2 + starting_balance="$(get_final_balance_from_statement "$latest_crunch_statement")" + else + starting_balance="${STARTING_BALANCE:-}" + if [ -n "$starting_balance" ]; then + echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$LAST_BALANCE" >&2 + exit 1 + fi + fi + echo "starting balance: $starting_balance" >&2 + echo "$starting_balance" +} + +generate_crunch_statements(){ + local passed_latest_statement=0 + local latest_crunch_statement + local starting_balance + latest_crunch_statement="$(get_latest_crunch_statement)" + starting_balance="$(get_starting_balance "$latest_crunch_statement")" + for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + if [ "$crunch_statement" = "$latest_crunch_statement" ]; then + passed_latest_statement=1 + fi + continue + fi + if [ $passed_latest_statement -lt 1 ]; then + continue + fi + "$converter" --credit-card --reverse-order --starting-balance "$starting_balance" "$statement" + starting_balance="$(get_final_balance_from_statement "$crunch_statement")" + done +} From bf1b85941ece805ed7b1dff66fd5a750b035a23d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:06:57 +0100 Subject: [PATCH 1324/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 9755b4435..1bf4d9e75 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.0' +__version__ = '0.7.1' class CrunchAccountingCsvStatementConverter(CLI): @@ -107,6 +107,7 @@ def run(self): log.info("converted '%s' => '%s'", filename, target_filename) else: log.error("FAILED to convert filename '%s'", filename) + sys.exit(2) log.info('Final Balance: {}'.format(self.running_balance)) def convert(self, filename, target_filename): From 651f4f9f14d1a310aa841a8f0c554681e1ca7d07 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:12:18 +0100 Subject: [PATCH 1325/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 1bf4d9e75..ab51cdac1 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.1' +__version__ = '0.7.2' class CrunchAccountingCsvStatementConverter(CLI): @@ -168,13 +168,17 @@ def detect_columns(self, csvreader): # want Transaction Date and not Posted Date if 'Date' in value and not 'Posted' in value: positions['date'] = position - elif 'Merchant Name' in value: - positions['desc'] = position # Original Amount column will be original currency eg 499 USD, but we only want native currency eg. 421.33 elif 'Amount' in value and not 'Original' in value: positions['amount'] = position elif 'Balance' in value: balance_position = position + # Barclaycard CSVs + elif 'Merchant Name' in value: + positions['desc'] = position + # Barclays CSVs + elif 'Memo' in value: + positions['desc'] = position for pos in positions: if positions[pos] is None: log.error('field %s not found', pos) From 7b9da2f721f4b379109cc9958ec109c88ea4f851 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:16:49 +0100 Subject: [PATCH 1326/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index ab51cdac1..4517f97b7 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.2' +__version__ = '0.7.3' class CrunchAccountingCsvStatementConverter(CLI): @@ -225,7 +225,7 @@ def validate_csvreader(csvreader, filename): # extra protection along the same lines as anti-json: # the first char of field should be alphanumeric, not syntax # however instead of isAlnum allow quotes for quoted CSVs to pass validation - if not isChars(field_list[0][0], 'A-Za-z0-9"'): + if field_list[0] != "" and not isChars(field_list[0][0], 'A-Za-z0-9"'): log.error('non-alphanumeric / quote opening character detected in CSV') return None count += 1 From 9971187d8671b6f3a3956dbbafb4c0382d47fa7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:22:04 +0100 Subject: [PATCH 1327/2779] updated lib.sh --- .../lib.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh index f6079e7fb..d6c217880 100755 --- a/crunch_accounting_csv_statement_converter_scripts/lib.sh +++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh @@ -30,7 +30,7 @@ get_latest_crunch_statement(){ latest_crunch_statement="$crunch_statement" fi done - if [ -n "$latest_crunch_statement" ]; then + if [ -n "${latest_crunch_statement:-}" ]; then echo "$latest_crunch_statement" fi } @@ -46,13 +46,15 @@ get_final_balance_from_statement(){ get_starting_balance(){ local starting_balance + local latest_crunch_statement="$1" if [ -n "${latest_crunch_statement:-}" ]; then echo "latest crunch statement is $latest_crunch_statement" >&2 starting_balance="$(get_final_balance_from_statement "$latest_crunch_statement")" else + echo "no latest crunch statement, getting starting balance from environment variable \$STARTING_BALANCE" >&2 starting_balance="${STARTING_BALANCE:-}" - if [ -n "$starting_balance" ]; then - echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$LAST_BALANCE" >&2 + if [ -z "$starting_balance" ]; then + echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$STARTING_BALANCE" >&2 exit 1 fi fi @@ -61,15 +63,20 @@ get_starting_balance(){ } generate_crunch_statements(){ + # only generate statements newer than the last generated one which provides the starting balance local passed_latest_statement=0 local latest_crunch_statement local starting_balance latest_crunch_statement="$(get_latest_crunch_statement)" + if [ -z "$latest_crunch_statement" ]; then + passed_latest_statement=1 + fi starting_balance="$(get_starting_balance "$latest_crunch_statement")" for statement in $STATEMENT_GLOB; do crunch_statement="${statement%.csv}_crunch.csv" if [ -f "$crunch_statement" ]; then - if [ "$crunch_statement" = "$latest_crunch_statement" ]; then + if [ $passed_latest_statement = 0 ] && + [ "$crunch_statement" = "$latest_crunch_statement" ]; then passed_latest_statement=1 fi continue From 8ac9b8f0ebd46a5732f9a2490364065910d0467f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:23:48 +0100 Subject: [PATCH 1328/2779] added barclays_crunch_accounting_csv_statement_converter.sh --- ...unch_accounting_csv_statement_converter.sh | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh diff --git a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh new file mode 100755 index 000000000..f667216a6 --- /dev/null +++ b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +. "$srcdir/lib.sh" + +# statements should be named in format: Barclays_Statement_YYYY-MM-DD.csv +export STATEMENT_GLOB="Barclays_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" + +# Barclays CSV statements often have whitespace starting fields instead of blank or 'null' +# unfortunately this resets all the original CSV timestamps each run so it's best to do only where needed +# UPDATE: no longer necessary, converter just ignores these blank fields now in the validation +#for statement in $STATEMENT_GLOB; do + #perl -pi -e 's/^\s+,/,/' "$statement" +#done + +generate_crunch_statements From 0a41818ffc74ba69fafd703e4f94b2be54a76732 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:27:34 +0100 Subject: [PATCH 1329/2779] updated lib.sh --- crunch_accounting_csv_statement_converter_scripts/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh index d6c217880..65a21926b 100755 --- a/crunch_accounting_csv_statement_converter_scripts/lib.sh +++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh @@ -84,7 +84,7 @@ generate_crunch_statements(){ if [ $passed_latest_statement -lt 1 ]; then continue fi - "$converter" --credit-card --reverse-order --starting-balance "$starting_balance" "$statement" + "$converter" "$@" --starting-balance "$starting_balance" "$statement" starting_balance="$(get_final_balance_from_statement "$crunch_statement")" done } From ca99e1ab3b47d5a98401a621e87d403c053a6998 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:27:38 +0100 Subject: [PATCH 1330/2779] updated barclaycard_crunch_accounting_csv_statement_converter.sh --- .../barclaycard_crunch_accounting_csv_statement_converter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh index cbf3568e3..708ab7a66 100755 --- a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh +++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh @@ -23,4 +23,4 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv export STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" -generate_crunch_statements +generate_crunch_statements --credit-card --reverse-order From cf9da7a9e888d9d2d42989c73f31089c64e2b8e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:27:42 +0100 Subject: [PATCH 1331/2779] updated barclays_crunch_accounting_csv_statement_converter.sh --- .../barclays_crunch_accounting_csv_statement_converter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh index f667216a6..704b9defd 100755 --- a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh +++ b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh @@ -30,4 +30,4 @@ export STATEMENT_GLOB="Barclays_Statement_[[:digit:]][[:digit:]][[:digit:]][[:di #perl -pi -e 's/^\s+,/,/' "$statement" #done -generate_crunch_statements +generate_crunch_statements --reverse-order From 825738646eb9081de400db1800f7a5bd076e86cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 29 May 2020 15:45:50 +0100 Subject: [PATCH 1332/2779] updated crunch_accounting_csv_statement_converter.py --- crunch_accounting_csv_statement_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 4517f97b7..102b84058 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -182,7 +182,7 @@ def detect_columns(self, csvreader): for pos in positions: if positions[pos] is None: log.error('field %s not found', pos) - return False + sys.exit(1) if balance_position is None and self.running_balance is None: self.usage('no balance column detected, please specify --starting-balance') return (positions, balance_position) From 248b6b1dbb11448e4cefd9836a34566729efad00 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 30 May 2020 18:55:11 +0100 Subject: [PATCH 1333/2779] updated buddy.yml --- buddy.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/buddy.yml b/buddy.yml index 8257e4b8d..533d34910 100644 --- a/buddy.yml +++ b/buddy.yml @@ -27,10 +27,14 @@ working_directory: "/buddy/devops-python-tools" docker_image_name: "library/ubuntu" docker_image_tag: "18.04" - setup_commands: + #setup_commands: + # this step gets cached, which results in + # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? + # - apt update + # - apt install -qy git make + execute_commands: - apt update - apt install -qy git make - execute_commands: - make init - make ci test volume_mappings: From d5e5da0614337498cc9278ddffe6ad1768b2208e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 30 May 2020 18:55:17 +0100 Subject: [PATCH 1334/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3f3c2f918..d8957934d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3f3c2f918b242e32a112a182ac461ab66331bab0 +Subproject commit d8957934d680bbf1fae688b7ee8cb543d1b65e0b From 2d5fd10c50ba1bc6c1fa7183c52925aaf7caba53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 30 May 2020 18:56:26 +0100 Subject: [PATCH 1335/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 93c911150..5e12d6ac8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 93c9111508dbbb756f99c8062c095580cf6bc203 +Subproject commit 5e12d6ac8a60d4eb90932a3b97773809eb70a754 From 68a9bb621af1ce02b72ec3e93a57fc00c1f9564a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 14:43:38 +0100 Subject: [PATCH 1336/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d8957934d..e3048bd4b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d8957934d680bbf1fae688b7ee8cb543d1b65e0b +Subproject commit e3048bd4be961071db74cd302a565fdac06d8ce2 From fc073dd92ec097535c5882c4e59ca10217ff92ac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 14:43:38 +0100 Subject: [PATCH 1337/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5e12d6ac8..56bf21007 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5e12d6ac8a60d4eb90932a3b97773809eb70a754 +Subproject commit 56bf21007849feeb15f4cb056a52b55deebda604 From f1cde920455d2fa5695d4077488aedb3fd5d9c5c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 15:56:13 +0100 Subject: [PATCH 1338/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e3048bd4b..1152d2f83 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e3048bd4be961071db74cd302a565fdac06d8ce2 +Subproject commit 1152d2f83b34e2adbbfa278188eef5ee0f6bae55 From f76040271560c9cc03e5be3c760b4154066c7868 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 15:56:13 +0100 Subject: [PATCH 1339/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 56bf21007..3933b12fe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 56bf21007849feeb15f4cb056a52b55deebda604 +Subproject commit 3933b12fef600a07959df49732714c614721bc12 From 24a0c3680c3925ff85a4a15b8caf26a8ba3d54fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 17:07:06 +0100 Subject: [PATCH 1340/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1152d2f83..408468520 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1152d2f83b34e2adbbfa278188eef5ee0f6bae55 +Subproject commit 408468520d273f36a827a1b9d62158a3d4db7ff4 From b1b7778c015cfcab01dae2a1e2cbc54fe0fa6351 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 Jun 2020 17:07:06 +0100 Subject: [PATCH 1341/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3933b12fe..f87fd873b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3933b12fef600a07959df49732714c614721bc12 +Subproject commit f87fd873b84579696bfb90fb46ee66643d84ecfb From e43d1f07d81882776319e1c22dc7235d519123b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Jun 2020 12:08:25 +0100 Subject: [PATCH 1342/2779] updated .appveyor.yml --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 7540e70a9..4488bbf71 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -61,8 +61,8 @@ install: - sudo apt purge -qy --allow-change-held-packages mssql-server # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages -- sudo apt update -qq -- sudo apt install -qy git make +- for x in `seq 10`; do sudo apt update -qq && break; sleep 60; done +- for x in `sed 10`; do sudo apt install -qy git make && break; sleep 60; done - make test_script: From 5546269378424a87b6bfdfbb81ca153b5e757a14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Jun 2020 12:10:40 +0100 Subject: [PATCH 1343/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 408468520..1dd707e62 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 408468520d273f36a827a1b9d62158a3d4db7ff4 +Subproject commit 1dd707e6228c9bda2bad2a684dfc3697237ed0b0 From e04d86e4450c4d313b66e2733d49494aaddaefda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Jun 2020 12:10:40 +0100 Subject: [PATCH 1344/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f87fd873b..9d3c31caf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f87fd873b84579696bfb90fb46ee66643d84ecfb +Subproject commit 9d3c31caf3d3efb90866ad225d3d469263d962a2 From 2d4c4819871e6ea8fc2b8e565ea610b1318cdbc9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 3 Jun 2020 23:46:21 +0100 Subject: [PATCH 1345/2779] updated test_anonymize.py --- tests/test_anonymize.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index 6850e650b..4bd3aebc0 100755 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -34,11 +34,11 @@ def run(): global dest # pylint: disable=global-statement src = {int(k) : v for k, v in src.items()} dest = {int(k) : v for k, v in dest.items()} - src_keys = [key for key in sorted(src)] # pylint: disable=redefined-outer-name - test_input = '\n'.join([src[key] for key in src_keys]) + src_keys = sorted(src) + test_input = '\n'.join([src[_] for _ in src_keys]) print('running anonymize tests using: {} {}'.format(anonymize, args)) - cmd = [anonymize] + [_ for _ in args.split()] + cmd = [anonymize] + args.split() process = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE) # encode as bytes for Python 3 :-/ test_input = str.encode(test_input, 'utf-8') @@ -46,7 +46,8 @@ def run(): index = 0 # convert bytes to string stdout = stdout.decode("utf-8") - for line in stdout.split('\n'): # pylint: disable=redefined-outer-name + # pylint: disable=redefined-outer-name + for line in stdout.split('\n'): key = src_keys[index] _input = src[key] expected = dest[key] From 72ca6a6ea61d3f4fdf937cb82d7eda00be9a15e7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 4 Jun 2020 14:57:33 +0100 Subject: [PATCH 1346/2779] updated anonymize.py --- anonymize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/anonymize.py b/anonymize.py index d46b6c796..10681eac0 100755 --- a/anonymize.py +++ b/anonymize.py @@ -81,6 +81,8 @@ ip_regex, \ subnet_mask_regex, \ user_regex + # used dynamically + # pylint: disable=unused-import from harisekhon.utils import \ domain_regex, \ email_regex, \ From 5179ca27cecf1d5b801a1ae37d1efda5f460f0c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 4 Jun 2020 15:01:25 +0100 Subject: [PATCH 1347/2779] updated anonymize.py --- anonymize.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/anonymize.py b/anonymize.py index 10681eac0..bf87520a6 100755 --- a/anonymize.py +++ b/anonymize.py @@ -357,8 +357,8 @@ def __init__(self): 'group2': r'({group_name}{sep}){user}'.format(group_name=group_name, sep=arg_sep, user=user_regex), 'group3': r'for\s+group\s+{group}'.format(group=user_regex), 'group4': r'(["\']{group_name}["\']\s*:\s*["\']?){group}'.format(group_name=group_name, group=user_regex), - 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format(\ - group='({}/)*{}'.format(user_regex, user_regex)), + 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format( + group='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)), 'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/(home|user)/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), @@ -370,7 +370,8 @@ def __init__(self): 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'\ .format(user_name=user_name, user=user_regex), #'user8': r'arn:aws:iam::\d{12}:user/{user}'.format(user=user_regex), - 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format(user='({}/)*{}'.format(user_regex, user_regex)), + 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format( + user='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)), 'password': r'([\.-]?{pass_word_phrase}{sep}){pw}'\ .format(pass_word_phrase=pass_word_phrase, sep=arg_sep, @@ -658,6 +659,7 @@ def process_options(self): self.anonymizations['ip'] = False else: for _ in self.anonymizations: + # pylint: disable=no-else-continue if _ in ('subnet_mask', 'mac', 'group'): continue elif _ == 'database': @@ -891,8 +893,7 @@ def anonymize(self, line): if line is None: if method: raise AssertionError('anonymize_{} returned None'.format(_)) - else: - raise AssertionError('anonymize_dynamic({}, line)'.format(_)) + raise AssertionError('anonymize_dynamic({}, line)'.format(_)) line += line_ending return line From 02d0ddd2fb16e6974f2dc80a0e84d635c4883222 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 13:58:05 +0100 Subject: [PATCH 1348/2779] updated config.yml --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 114368b87..9b803fbad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,10 +30,12 @@ jobs: # so using harisekhon/dev:ubuntu instead of base ubuntu image #image: harisekhon/dev:ubuntu steps: - # to allow docker networking to work - - run: sudo sysctl net.ipv4.ip_forward=1 - - run: sudo service docker restart - checkout + - run: for x in `seq 10`; do sudo apt update -q && break; sleep 60; done + - run: for x in `seq 10`; do sudo apt install -qy git make && break; sleep 60; done - run: make init - run: make + # to allow docker networking to work + - run: sudo sysctl net.ipv4.ip_forward=1 + - run: sudo service docker restart - run: make test From 9b80a6578b62cff0d80b2ae5d31bf11a3fdcde69 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 14:23:28 +0100 Subject: [PATCH 1349/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 4cfd2cb8e..89c409ada 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -30,12 +30,12 @@ blocks: #execution_time_limit: # hours: 2 task: - env_vars: + #env_vars: # $PATH selects /usr/bin/python and /usr/local/bin/pip which are mismatched versions of Python - - name: PYTHON - value: python3 - - name: PIP - value: pip3 + #- name: PYTHON + # value: python3 + #- name: PIP + # value: pip3 prologue: commands: - cache restore @@ -57,10 +57,10 @@ blocks: when: "branch = 'master'" task: # because otherwise on Mac it uses /usr/bin/python (2.7) but /usr/local/bin/pip (python 3.8) - env_vars: + #env_vars: # to match /usr/local/bin/pip version from $PATH - - name: PYTHON - value: python3 + #- name: PYTHON + # value: python3 # must be quoted to force string, otherwise pipeline fails to run with this parsing error: # Error: [{"Type mismatch. Expected String but got Integer.", "#/blocks/1/task/env_vars/1/value"}] #- name: DEBUG @@ -89,6 +89,9 @@ blocks: - rbenv global system # also considered this: # - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done + # + # fix for python vs pip version mismatch + - ln -svf /usr/local/bin/python3 /usr/local/bin/python jobs: - name: build commands: From db9bee094377db3140add8230ee1231adb71e420 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:14:00 +0100 Subject: [PATCH 1350/2779] added ci_bootstrap.sh --- setup/ci_bootstrap.sh | 83 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 setup/ci_bootstrap.sh diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh new file mode 100644 index 000000000..dc2e22b00 --- /dev/null +++ b/setup/ci_bootstrap.sh @@ -0,0 +1,83 @@ +#!/bin/sh +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-06-02 17:43:35 +0100 (Tue, 02 Jun 2020) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +# Designed to bootstrap all CI systems with retries to make sure the networking, package lists and package repos works before proceeding +# +# Minimizes CI build failures due to temporary networking blips, which happens more often than you would think when you have a large number of CI builds across a lot of disparate systems + +set -eu +[ -n "${DEBUG:-}" ] && set -x + +max_tries=10 +interval=60 # secs + +sudo="" +# EUID undefined in posix sh +#[ $EUID = 0 ] || sudo=sudo +[ "$(whoami)" = root ] || sudo=sudo + +retry(){ + # no local in posix sh + count=0 + while true; do + # no let or bare (()) in posix sh, must discard output rather than execute it + _=$((count+=1)) + printf "%s try %d: " "$(date '+%F %T')" "$count" + echo "$*" + "$@" && + break; + echo + if [ $count -ge $max_tries ]; then + echo "$count tries failed, aborting..." + exit 1 + fi + echo "sleeping for $interval secs before retrying" + sleep "$interval" + echo + done +} + +if [ "$(uname -s)" = Darwin ]; then + echo "Bootstrapping Mac" + # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make) + #retry "$srcdir/install_homebrew.sh" + if command -v brew 2>&1; then + retry $sudo brew update + fi +elif [ "$(uname -s)" = Linux ]; then + echo "Bootstrapping Linux" + if type -P apk >/dev/null 2>&1; then + retry $sudo apk update + retry $sudo apk add --no-progress bash git make + elif type apt-get >/dev/null 2>&1; then + retry $sudo apt-get update -q + retry $sudo apt-get install -qy git make + elif type yum >/dev/null 2>&1; then + #retry $sudo yum makecache + retry $sudo yum install -qy git make + else + echo "Package Manager not found on Linux, cannot bootstrap" + exit 1 + fi +else + echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time" + exit 1 +fi + +#retry make init + +# not calling make because in some CI systems we call 'make ci' which includes retries but in others with more restrictive build minutes we only run 'make' for a single shot build +# +#make From ecea5c522ee9cf34c7a681aaf39ed1bcb2de504f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:57 +0100 Subject: [PATCH 1351/2779] updated Jenkinsfile --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 99d0ac0b7..c9fbad7d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,8 +70,7 @@ pipeline { // sh 'apt install -qy make' // sh 'make init' sh """ - apt update -q && - apt install -qy make && + setup/ci_bootstrap.sh && make init """ } From b99d70ea2c325322177742d80dc7f7f45b0f1d4e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:58 +0100 Subject: [PATCH 1352/2779] updated .appveyor.yml --- .appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 4488bbf71..eb43a0250 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -61,8 +61,7 @@ install: - sudo apt purge -qy --allow-change-held-packages mssql-server # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages -- for x in `seq 10`; do sudo apt update -qq && break; sleep 60; done -- for x in `sed 10`; do sudo apt install -qy git make && break; sleep 60; done +- setup/ci_bootstrap.sh - make test_script: From bcc02a8ecfe4e00301cd29c5ef09ec4b8f3d745b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:58 +0100 Subject: [PATCH 1353/2779] updated pipeline.yml --- .buildkite/pipeline.yml | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c8bfbe881..33f4e790a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-03-13 21:10:39 +0000 (Fri, 13 Mar 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -20,22 +20,9 @@ # - command: buildkite-agent pipeline upload steps: - - command: | - type make 2>/dev/null || - if type apk 2>/dev/null; then - apk add --no-cache --no-progress make - # apt is /usr/bin/apt - # Unable to locate an executable at "/Users/hari/.sdkman/candidates/java/current/bin/apt" (-1) - elif type apt-get 2>/dev/null; then - apt-get update -q && - apt-get install -qy make - elif type yum 2>/dev/null; then - rpm -q make || yum install -y make - elif type brew 2>/dev/null; then - brew install make - fi - label: install make - timeout: 20 # brew can take 10 mins just to do a brew update + - command: setup/ci_bootstrap.sh + label: ci bootstrap + timeout: 30 # brew can take 10 mins just to do a brew update - wait - command: make init label: init From bd18c97dadb091a4b6f74db39d1a54d29bc02b2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:58 +0100 Subject: [PATCH 1354/2779] updated config.yml --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b803fbad..69f7a6179 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,8 +31,7 @@ jobs: #image: harisekhon/dev:ubuntu steps: - checkout - - run: for x in `seq 10`; do sudo apt update -q && break; sleep 60; done - - run: for x in `seq 10`; do sudo apt install -qy git make && break; sleep 60; done + - run: setup/ci_bootstrap.sh - run: make init - run: make # to allow docker networking to work From 208abb3afeeaec7f3718b218d34d406edde5e830 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:58 +0100 Subject: [PATCH 1355/2779] updated .cirrus.yml --- .cirrus.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 38752ca97..46de8ac49 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 16:55:36 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -13,8 +13,13 @@ # https://www.linkedin.com/in/harisekhon # +# https://cirrus-ci.org/guide/writing-tasks/ + container: image: ubuntu:18.04 task: - script: apt update -qq && apt install -qy git make && make init && make ci test + script: + - setup/ci_bootstrap.sh + - make init + - make ci test From 4705d70fa8f9edef67a15b300aeb69c9011ed65b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:59 +0100 Subject: [PATCH 1356/2779] updated .concourse.yml --- .concourse.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.concourse.yml b/.concourse.yml index d50f7ecf4..2abb4ac17 100644 --- a/.concourse.yml +++ b/.concourse.yml @@ -53,7 +53,6 @@ jobs: - -c - | cd code && - apt update -q && - apt install -qy git make && + setup/ci_bootstrap.sh && make init && make ci test From 61f5eab2c816c31f56e2c1430fd078f97a97ad6d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:59 +0100 Subject: [PATCH 1357/2779] updated .drone.yml --- .drone.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 4dd6036f6..f57a8d178 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -23,8 +23,7 @@ steps: # environment: # DEBUG: 1 commands: - - apt update -qq - - apt install -qy git make + - setup/ci_bootstrap.sh - make init - make ci - make test From 931737931b814c27f8abd58b81e9d7856be61584 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:59 +0100 Subject: [PATCH 1358/2779] updated .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1613d1410..af2a6a22b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,6 +19,6 @@ image: ubuntu:18.04 job: before_script: - - apt-get update -qq && apt-get install -yq git make + - setup/ci_bootstrap.sh script: - make init && make ci test From c32cbdd454aba2b62eb806082e7e461df56b9226 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:37:59 +0100 Subject: [PATCH 1359/2779] updated .gocd.yml --- .gocd.yml | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/.gocd.yml b/.gocd.yml index ad1c3d693..54f7a3067 100644 --- a/.gocd.yml +++ b/.gocd.yml @@ -38,24 +38,30 @@ pipelines: type: success allow_only_on_success: false jobs: - apt-update: +# apt-update: +# timeout: 10 +# tasks: +# - exec: +# command: apt +# arguments: +# - update +# run_if: passed +# install-make: +# timeout: 10 +# tasks: +# - exec: +# command: apt +# arguments: +# - install +# - -qy +# - git +# - make +# run_if: passed + ci-bootstrap: timeout: 10 tasks: - exec: - command: apt - arguments: - - update - run_if: passed - install-make: - timeout: 10 - tasks: - - exec: - command: apt - arguments: - - install - - -qy - - git - - make + command: setup/ci_bootstrap.sh run_if: passed init: timeout: 10 From 5f7849f1c6dd115b92659c50dd85a3c3f3975f13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:00 +0100 Subject: [PATCH 1360/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 89c409ada..09b556190 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,8 +44,7 @@ blocks: - name: build commands: - checkout - - sudo apt update -qq - - sudo apt install -qy git make + - setup/ci_bootstrap.sh - make init - make ci - make test From 1e056326b0a04d15056a601ff1405a704e3633ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:00 +0100 Subject: [PATCH 1361/2779] updated azure-pipelines.yml --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cdf98c65a..0850b087f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -38,5 +38,6 @@ pool: # hacky workaround to Azure Pipelines limitations :-( steps: -- script: sudo docker run -v "$PWD":/pwd ubuntu:18.04 /bin/bash -c "set -ex && cd /pwd && apt update -qq && apt install -qy git make && make init && make ci test" +- script: sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c 'set -ex && cd /code && setup/ci_bootstrap.sh && make init && make ci test' + displayName: docker build From c1fcb42ef589f0d3041ce1f4ad86590070e72e74 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:00 +0100 Subject: [PATCH 1362/2779] updated bitbucket-pipelines.yml --- bitbucket-pipelines.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index f44aee45d..a7ec1b183 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 17:08:57 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -23,4 +23,7 @@ pipelines: default: - step: script: - - apt update -qq && apt install -qy git make && make init && make ci test + - setup/ci_bootstrap.sh + - make init + - make ci + - make test From fe3d147ffeae2b83816cf6f0cd16854936d3909b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:00 +0100 Subject: [PATCH 1363/2779] updated buddy.yml --- buddy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buddy.yml b/buddy.yml index 533d34910..aed9fc3bc 100644 --- a/buddy.yml +++ b/buddy.yml @@ -33,10 +33,10 @@ # - apt update # - apt install -qy git make execute_commands: - - apt update - - apt install -qy git make + - setup/ci_bootstrap.sh - make init - - make ci test + - make ci + - make test volume_mappings: - "/:/buddy/devops-python-tools" shell: "BASH" From 32ffbefca1731eabcbf00a18a007aaa8b5a46faa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:01 +0100 Subject: [PATCH 1364/2779] updated codefresh.yml --- codefresh.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codefresh.yml b/codefresh.yml index 70990129d..542c323d8 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 17:43:07 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -33,7 +33,7 @@ steps: arguments: image: 'ubuntu:18.04' commands: - - apt update -qq && apt install -qy git make + - setup/ci_bootstrap.sh - make init - make ci - make test From f12a652de2747653a75fc4da587b3dc24eb7e538 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:01 +0100 Subject: [PATCH 1365/2779] updated shippable.yml --- shippable.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shippable.yml b/shippable.yml index 43f564753..0c19cb053 100644 --- a/shippable.yml +++ b/shippable.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -33,6 +33,7 @@ build: - rm -fv /etc/apt/sources.list.d/cassandra.sources.list* - rm -fv /etc/apt/sources.list.d/yarn.list* #- shippable_retry make + - setup/ci_bootstrap.sh - make init - make ci - make test From d3f580ea68ef3a99d6798acc2d54f21703edad06 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:38:01 +0100 Subject: [PATCH 1366/2779] updated wercker.yml --- wercker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wercker.yml b/wercker.yml index 18b14ed8f..f57ab989f 100644 --- a/wercker.yml +++ b/wercker.yml @@ -2,7 +2,7 @@ # Author: Hari Sekhon # Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020) # -# vim:ts=4:sts=4:sw=4:et +# vim:ts=2:sts=2:sw=2:et # # https://github.com/harisekhon/devops-python-tools # @@ -20,8 +20,8 @@ box: debian build: steps: - script: - name: install git & make - code: apt-get update -qq && apt-get install -qy git make + name: ci bootstrap + code: setup/ci_bootstrap.sh - script: name: init code: make init From 07b68156b6cf929d9e0e265635cabe72a70bbd29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:46:10 +0100 Subject: [PATCH 1367/2779] added boot --- boot | 1 + 1 file changed, 1 insertion(+) create mode 120000 boot diff --git a/boot b/boot new file mode 120000 index 000000000..4092e5539 --- /dev/null +++ b/boot @@ -0,0 +1 @@ +setup/bootstrap.sh \ No newline at end of file From b6c79d0b34a4ac05399901fcaf37519c0f99d141 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:47:22 +0100 Subject: [PATCH 1368/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1dd707e62..bae152292 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1dd707e6228c9bda2bad2a684dfc3697237ed0b0 +Subproject commit bae152292cd75f72e92bce6131ff9a6b5ad42cc3 From 58706abe890d0eb04ebbffed97c23fa9e1756169 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:47:22 +0100 Subject: [PATCH 1369/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9d3c31caf..04ba0ca02 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9d3c31caf3d3efb90866ad225d3d469263d962a2 +Subproject commit 04ba0ca02b2196b45a0dc7f0d29e3d59642fd073 From a179e1e49acbdd238fc6b07f5988a177891b9d16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 17:59:05 +0100 Subject: [PATCH 1370/2779] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 setup/ci_bootstrap.sh diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh old mode 100644 new mode 100755 From 1dd1ce2ee77f0695ee914f198159f8ad32493105 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Jun 2020 18:00:26 +0100 Subject: [PATCH 1371/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 04ba0ca02..312908be6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 04ba0ca02b2196b45a0dc7f0d29e3d59642fd073 +Subproject commit 312908be6b409520e2ce51d05c6fc5779b0dbaf5 From a076ab35d703ef89d542487109443dabd2444509 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 Jun 2020 12:57:21 +0100 Subject: [PATCH 1372/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bae152292..f4641004f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bae152292cd75f72e92bce6131ff9a6b5ad42cc3 +Subproject commit f4641004f24d8ae99ffef0e170185350b2d391f1 From 867d9c17e5a24f7e052fc16fcb46bfc1b31d13a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 Jun 2020 12:57:21 +0100 Subject: [PATCH 1373/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 312908be6..d47683229 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 312908be6b409520e2ce51d05c6fc5779b0dbaf5 +Subproject commit d47683229b7a10ae74c4a8315befcb705c31c96a From 3aeee83c9444d5f812350fc7453d77e164144755 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 7 Jun 2020 15:17:54 +0100 Subject: [PATCH 1374/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d47683229..7050d3664 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d47683229b7a10ae74c4a8315befcb705c31c96a +Subproject commit 7050d3664e3d64dc4a6cb3fbb3a469e899a00f92 From d9f139f00678691436fdeb5a2b0b43d1b328474c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 15 Jun 2020 15:13:24 +0100 Subject: [PATCH 1375/2779] updated find_duplicate_files.py --- find_duplicate_files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index b1f35742e..cbf7b082b 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -210,27 +210,27 @@ def run(self): for filepath in sorted(self.dup_filepaths): print(filepath) sys.exit(4) - print('Duplicates detected!\n') + print('Duplicates detected!') if self.dups_by_name: - print('Duplicates by name:\n') + print('\nDuplicates by name:\n') for basename in self.dups_by_name: print("--\nbasename '{0}':".format(basename)) for filepath in sorted(self.dups_by_name[basename]): print(filepath) if self.dups_by_size: - print('Duplicates by size:\n') + print('\nDuplicates by size:\n') for size in self.dups_by_size: print("--\nsize '{0}' bytes:".format(size)) for filepath in sorted(self.dups_by_size[size]): print(filepath) if self.dups_by_hash: - print('Duplicates by checksum:\n') + print('\nDuplicates by checksum:\n') for checksum in self.dups_by_hash: print("--\nchecksum '{0}':".format(checksum)) for filepath in sorted(self.dups_by_hash[checksum]): print(filepath) if self.dups_by_regex: - print('Duplicates by regex match ({0}):\n'.format(self.regex)) + print('\nDuplicates by regex match ({0}):\n'.format(self.regex)) for matching_portion in self.dups_by_regex: print("--\nregex matching portion '{0}':".format(matching_portion)) for filepath in sorted(self.dups_by_regex[matching_portion]): From 0ea610400a05c550dd2ade0bdc5c3bf14dcabeec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 7 Jul 2020 14:00:33 +0100 Subject: [PATCH 1376/2779] updated find_duplicate_files.py --- find_duplicate_files.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index cbf7b082b..9fa274f26 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -35,6 +35,8 @@ then will treat the entire regex as the capture. Regex is case insensitive by default and applies only to the file's basename +Exits with exit code 4 if duplicates are found + Can restrict methods of finding duplicates to any combination of --name / --size / --checksum (checksum implies size as an efficiency shortcut) / --regex. If none are specified then will try name, size + checksum. If specifying any one of these options then the others will not run unless also explicitly specified. From e57b02628a4ec1fdd91bccc358190c5bc4f2c346 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 18 Jul 2020 17:38:07 +0100 Subject: [PATCH 1377/2779] updated find_duplicate_files.py --- find_duplicate_files.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 9fa274f26..4e382fa27 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -49,10 +49,12 @@ - By default this program will short-circuit to stop processing a file as soon as it is determined to be a duplicate file via one of the above methods in that order for efficiency. This means that if 2 files have duplicate names, and a third has a different name but the same checksum as the second one, the second one's size + checksum will not have -been recorded stored and so the third duplicate will not be detected. However, if you removed one duplicate the next -run of this program would find the other duplicate via the other dimension of checking. Given it's a rare condition -it's probably not worth the extra overhead in everyday use but this behaviour can be overridden by specifying the ---no-short-circuit option too run every check on every file. Be aware this will slow down the process. +been checked and so a third duplicate with a different name will not be detected by size / checksum. In most cases this +is a good thing to finish quicker and avoid unnecessary checksumming which is computationally expensive and time +consuming for large files. If you remove one duplicate then the next run of this program would find the other +duplicate via the additional checks of size and checksumming. Given it's a rare condition it's probably not worth the +extra overhead in everyday use but this behaviour can be overridden by specifying the --no-short-circuit option to run +every check on every file. Be aware this will slow down the process. To see progress of which files are matching size, backtracking to hash them for comparison etc use --verbose twice or -vv. To see which files are being checked use triple verbose mode -vvv From 748630ae42cbb9d85d6dce305ae3143d9db79450 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 23 Jul 2020 12:54:25 +0100 Subject: [PATCH 1378/2779] updated find_duplicate_files.py --- find_duplicate_files.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 4e382fa27..6f235fadb 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -41,6 +41,9 @@ an efficiency shortcut) / --regex. If none are specified then will try name, size + checksum. If specifying any one of these options then the others will not run unless also explicitly specified. +If you want to find files that are probably the same by byte count but may not have the same checksum due to minor +corruption, such as large media files, then specify --size but do not specify --checksum which supercedes it + Caveats: - The limitation of the checksum approach is that it can't determine files as duplicates if there is any From 4302de0fe0e18d35e237eed93da382c8fc6be74e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:39:56 +0100 Subject: [PATCH 1379/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 305b8b1d6..89369eb58 100644 --- a/README.md +++ b/README.md @@ -427,7 +427,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 200+ DevOps Bash scripts, advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.toprc`, Utility Code Library used by CI and all my GitHub repos - includes code for AWS, Kubernetes, Kafka, Docker, Git, Code & build linting, package management for Linux / Mac / Perl / Python / Ruby / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 300+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Kafka, Docker, Hadoop, SQL, Athena, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, Spotify API & MP3 tools, Git, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies * [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... From 2b776e2864bddb83a7cda345a0f1294319e60c27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:40:00 +0100 Subject: [PATCH 1380/2779] updated anonymize.py --- anonymize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/anonymize.py b/anonymize.py index bf87520a6..339376c13 100755 --- a/anonymize.py +++ b/anonymize.py @@ -83,6 +83,7 @@ user_regex # used dynamically # pylint: disable=unused-import + # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend from harisekhon.utils import \ domain_regex, \ email_regex, \ From a60f16545e31f2056d696506c091e313072665ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:41:07 +0100 Subject: [PATCH 1381/2779] updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 89369eb58..f146e8e6c 100644 --- a/README.md +++ b/README.md @@ -424,7 +424,6 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### - * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) * [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 300+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Kafka, Docker, Hadoop, SQL, Athena, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, Spotify API & MP3 tools, Git, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies From e5c8797ecd284e081e51a9d26ed678b33aa7af66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:43:47 +0100 Subject: [PATCH 1382/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f4641004f..853b19466 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f4641004f24d8ae99ffef0e170185350b2d391f1 +Subproject commit 853b194664b69a12321c70a3ef3d41e190dd0ed9 From 5fdf345ef903f299f9eb69c362c997b36abb7bac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 25 Jul 2020 15:43:47 +0100 Subject: [PATCH 1383/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7050d3664..fc127555b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7050d3664e3d64dc4a6cb3fbb3a469e899a00f92 +Subproject commit fc127555b0d8163e9949fada918fbc3f1a1f5745 From 6423b3808652936407929c1266a673b5e4d08d68 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 11:37:45 +0100 Subject: [PATCH 1384/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f146e8e6c..208c2780f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/) [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools.svg)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) +[![StarTrack](https://img.shields.io/badge/Star-Track-blue)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) From 99b24cab83c8b510e110849f78da1cbe9ab6adf5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 11:40:30 +0100 Subject: [PATCH 1385/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 853b19466..b9ad2ed08 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 853b194664b69a12321c70a3ef3d41e190dd0ed9 +Subproject commit b9ad2ed08f5474ca13d9cef7d60c44c34d467a46 From 8d096756919af6e0c5e4608e44352fbfb7acc2b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 11:40:30 +0100 Subject: [PATCH 1386/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fc127555b..55a3bc182 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fc127555b0d8163e9949fada918fbc3f1a1f5745 +Subproject commit 55a3bc1825f3433c115b759dc5ca7775933419f4 From 7a845b9395aee8083f9770e40af5234c2c94892e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 17:56:57 +0100 Subject: [PATCH 1387/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b9ad2ed08..b24bcf168 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b9ad2ed08f5474ca13d9cef7d60c44c34d467a46 +Subproject commit b24bcf168d8e95feb7a2b11a83cb6ddf85543d12 From 70182f306f969e17582b14fb9d48eb9c4a3bda75 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 Jul 2020 17:56:57 +0100 Subject: [PATCH 1388/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 55a3bc182..12a344a0b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 55a3bc1825f3433c115b759dc5ca7775933419f4 +Subproject commit 12a344a0b686dd6a2618a92f7d8d6311aed73037 From 7c12c396c193e1f0f69a7154ca96d3828d207755 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 31 Jul 2020 14:32:19 +0100 Subject: [PATCH 1389/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b24bcf168..2f58c3492 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b24bcf168d8e95feb7a2b11a83cb6ddf85543d12 +Subproject commit 2f58c3492e68c7ab1f778197a451fe892827df8f From d856c134918baad561492ed9005d923522ecb686 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 31 Jul 2020 14:32:20 +0100 Subject: [PATCH 1390/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 12a344a0b..8a8ed26f1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 12a344a0b686dd6a2618a92f7d8d6311aed73037 +Subproject commit 8a8ed26f1250132446a54f0db627bd14c214bfe4 From 54d86180a7fcb34c5f2656c8135a049b325444d7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 31 Jul 2020 15:58:30 +0100 Subject: [PATCH 1391/2779] added find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 206 ++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100755 find_missing_files_in_sequence.py diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py new file mode 100755 index 000000000..6261932c5 --- /dev/null +++ b/find_missing_files_in_sequence.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-07-31 11:03:17 +0100 (Fri, 31 Jul 2020) +# +# https://github.com/harisekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/harisekhon +# + +""" + +Finds missing files by numeric sequence, assuming a uniformly numbered file naming convention across files + +Files / directories are given as arguments or via standard input + +Directories are recursed + +You must only give directories that should be sharing a contiguously numbered file naming convention for each +single run of this tool + +Accounts for zero padding in numbered files + +Caveats: + +- This is more complicated than you'd first think as there are so many file naming variations so this is not the most + universally bulletproof piece of code in this repo by a long shot and may require advanced tuning --regex tuning to + match your use case + +- Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be. + If you are looking for missing MP3 files, then you might be able to use 'mediainfo' to get the max track position and + see if the files go that high + +- Returns globs instead of explicit missing filenames since suffixes can vary after numbers. If you have a simple enough + use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print the missing + files more explicitly, but in the general case you cannot account for suffix naming that isn't consistent, such as + chapters of audiobooks eg. + + 'blah 01 - chapter 1.mp3' + 'blah 02 - chapter 2.mp3' + + so in the general case you cannot always infer suffixes, hence why it is left as globs. + Simpler single use case scripts would be better for printing explicit missing files as they can use hardcoded suffixes + +- Doesn't currently find entire missing CD / disks in the naming format, but you should be able to see those cases + easily by eye + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import glob +#import logging +import os +import re +import sys +import traceback +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + from harisekhon.utils import log, log_option, validate_regex, isFloat, UnknownError + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class FindMissingFiles(CLI): + + def __init__(self): + # Python 2.x + super(FindMissingFiles, self).__init__() + # Python 3.x + # super().__init__() + self.paths = [] + self.regex_default = r'(? 0: + self.paths = self.args + else: + self.paths = sys.stdin.readlines() + log_option('paths', self.paths) + + def is_included(self, path): + if not self.include: + return True + if self.include.search(path): + log.debug("including path: %s", path) + return True + return False + + def is_excluded(self, path): + if not self.exclude: + return False + if self.exclude.search(path): + log.debug("excluding path: %s", path) + return True + return False + + def run(self): + for path in self.paths: + if self.is_excluded(path): + continue + if not self.is_included(path): + continue + if not os.path.exists(path): + raise UnknownError('path not found: {}'.format(path)) + if os.path.isdir(path): + self.process_directory(directory=path) + elif os.path.isfile(path): + self.check_file(filename=path) + + def process_directory(self, directory): + for root, dirs, files in os.walk(directory, topdown=True): + for filename in files: + file_path = os.path.join(root, filename) + if not self.is_included(file_path): + continue + if self.is_excluded(file_path): + continue + self.check_file(filename=file_path) + for dirname in dirs: + dir_path = os.path.join(root, dirname) + if not self.is_included(dir_path): + continue + if self.is_excluded(dir_path): + continue + self.process_directory(directory=dir_path) + + def check_file(self, filename): + log.debug('checking file \'%s\'', filename) + match = self.regex.search(filename) + if not match: + log.debug('failed to match file \'%s\', skipping...', filename) + return + # will error out here if you've supplied your own regex without capture brackets + # or if you've got pre-captures - let this bubble to user to fix their regex + file_prefix = match.group(1) + file_number = match.group(2) + if not isFloat(file_number): + raise UnknownError('regex captured non-float for filename: {}'.format(filename)) + if file_prefix is None: + file_prefix = '' + padding = len(file_number) + file_number = int(file_number) + self.determine_missing_file_backfill(file_prefix, file_number, padding) + + def determine_missing_file_backfill(self, file_prefix, file_number, padding): + if file_number != 1: + file_number -= 1 + expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() + expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) + if not glob.glob(expected_last_filename_glob): + # by recursing and printing on the unwind we get correctly ordered file numbering ascending + self.determine_missing_file_backfill(file_prefix, file_number, padding) + print(expected_last_filename_glob) + + +if __name__ == '__main__': + FindMissingFiles().main() From 251bb73fd3f739200567ccc047a52296550b033a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 31 Jul 2020 16:18:12 +0100 Subject: [PATCH 1392/2779] added --fixed-suffix support --- find_missing_files_in_sequence.py | 73 +++++++++++++++++++------------ 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 6261932c5..29bf95502 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -21,9 +21,9 @@ Files / directories are given as arguments or via standard input -Directories are recursed +Directories are recursed and their files examined for missing numbers before each one -You must only give directories that should be sharing a contiguously numbered file naming convention for each +Only supply files / directories that should be sharing a contiguously numbered file naming convention in each single run of this tool Accounts for zero padding in numbered files @@ -35,19 +35,21 @@ match your use case - Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be. - If you are looking for missing MP3 files, then you might be able to use 'mediainfo' to get the max track position and - see if the files go that high + If you are looking for missing MP3 files, then you might be able to use 'mediainfo' to get the total number of tracks + and see if the files go that high -- Returns globs instead of explicit missing filenames since suffixes can vary after numbers. If you have a simple enough - use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print the missing - files more explicitly, but in the general case you cannot account for suffix naming that isn't consistent, such as - chapters of audiobooks eg. +- Returns globs by default instead of explicit missing filenames since suffixes can vary after numbers. If you have a + simple enough use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print + the missing files more explicitly, but in the general case you cannot account for suffix naming that isn't consistent, + such as chapters of audiobooks eg. - 'blah 01 - chapter 1.mp3' - 'blah 02 - chapter 2.mp3' + 'blah 01 - chapter about X.mp3' + 'blah 02 - chapter about Y.mp3' - so in the general case you cannot always infer suffixes, hence why it is left as globs. - Simpler single use case scripts would be better for printing explicit missing files as they can use hardcoded suffixes + so in the general case you cannot always infer suffixes, hence why it is left as globs. If you are sure that the + suffixes don't change then you can specify --fixed-suffix and it will infer each file's suffix as the basis for any + numerically missing files in the sequence, but if used where this is not the case, it'll generate a lot of false + positives that the default globbing mode would have handled - Doesn't currently find entire missing CD / disks in the naming format, but you should be able to see those cases easily by eye @@ -76,7 +78,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2.0' class FindMissingFiles(CLI): @@ -87,10 +89,11 @@ def __init__(self): # Python 3.x # super().__init__() self.paths = [] - self.regex_default = r'(? 0: + if self.args: self.paths = self.args else: self.paths = sys.stdin.readlines() @@ -183,23 +187,34 @@ def check_file(self, filename): # or if you've got pre-captures - let this bubble to user to fix their regex file_prefix = match.group(1) file_number = match.group(2) + file_suffix = match.group(3) if not isFloat(file_number): raise UnknownError('regex captured non-float for filename: {}'.format(filename)) if file_prefix is None: file_prefix = '' + if file_suffix is None: + file_suffix = '' padding = len(file_number) file_number = int(file_number) - self.determine_missing_file_backfill(file_prefix, file_number, padding) + self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) - def determine_missing_file_backfill(self, file_prefix, file_number, padding): + def determine_missing_file_backfill(self, file_prefix, file_number, padding, file_suffix): if file_number != 1: file_number -= 1 - expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() - expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) - if not glob.glob(expected_last_filename_glob): - # by recursing and printing on the unwind we get correctly ordered file numbering ascending - self.determine_missing_file_backfill(file_prefix, file_number, padding) - print(expected_last_filename_glob) + if self.fixed_suffix: + explicit_last_filename = '{}{:0>%(padding)s}{}' % locals() + explicit_last_filename = explicit_last_filename.format(file_prefix, file_number, file_suffix) + if not os.path.isfile(explicit_last_filename): + # by recursing and printing on the unwind we get correctly ordered file numbering ascending + self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) + print(explicit_last_filename) + else: + expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() + expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) + if not glob.glob(expected_last_filename_glob): + # by recursing and printing on the unwind we get correctly ordered file numbering ascending + self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) + print(expected_last_filename_glob) if __name__ == '__main__': From ecc55bb9eb6fcfe5e165bd52ddbd0c007f7949e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:45:16 +0100 Subject: [PATCH 1393/2779] refactored out recursion of file backtracking for larger file lists to avoid recursion limit --- find_missing_files_in_sequence.py | 79 +++++++++++++++++++------------ 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 29bf95502..e51b1ab4e 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -30,13 +30,14 @@ Caveats: -- This is more complicated than you'd first think as there are so many file naming variations so this is not the most - universally bulletproof piece of code in this repo by a long shot and may require advanced tuning --regex tuning to - match your use case +- This is more complicated than you'd first think as there are so many file naming variations that no code could ever + be universally bulletproof and will require advanced regex tuning to match your use case (this is tuned for the most + common use cases I've found but yours may vary since there are huge file naming variations between people and + environments) - Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be. - If you are looking for missing MP3 files, then you might be able to use 'mediainfo' to get the total number of tracks - and see if the files go that high + If you are looking for missing MP3 files, then you might be able to check the mp3 tag metadata using programs like + 'mediainfo' to get the total number of tracks and see if the files go that high - Returns globs by default instead of explicit missing filenames since suffixes can vary after numbers. If you have a simple enough use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print @@ -71,16 +72,17 @@ libdir = os.path.join(srcdir, 'pylib') sys.path.append(libdir) try: - from harisekhon.utils import log, log_option, validate_regex, isFloat, UnknownError + from harisekhon.utils import log, log_option, validate_regex, isInt, UnknownError from harisekhon import CLI except ImportError as _: print(traceback.format_exc(), end='') sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.3.0' +# pylint: disable=too-many-instance-attributes class FindMissingFiles(CLI): def __init__(self): @@ -89,22 +91,34 @@ def __init__(self): # Python 3.x # super().__init__() self.paths = [] - self.regex_default = r'(? 1: + file_number = self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) + if self.missing_files: + print('\n'.join(reversed(self.missing_files))) + self.missing_files = [] def determine_missing_file_backfill(self, file_prefix, file_number, padding, file_suffix): - if file_number != 1: - file_number -= 1 - if self.fixed_suffix: - explicit_last_filename = '{}{:0>%(padding)s}{}' % locals() - explicit_last_filename = explicit_last_filename.format(file_prefix, file_number, file_suffix) - if not os.path.isfile(explicit_last_filename): - # by recursing and printing on the unwind we get correctly ordered file numbering ascending - self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) - print(explicit_last_filename) + file_number -= 1 + if self.fixed_suffix: + explicit_last_filename = '{}{:0>%(padding)s}{}' % {'padding': padding} + explicit_last_filename = explicit_last_filename.format(file_prefix, file_number, file_suffix) + if not os.path.isfile(explicit_last_filename): + self.missing_files.append(explicit_last_filename) else: - expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() - expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) - if not glob.glob(expected_last_filename_glob): - # by recursing and printing on the unwind we get correctly ordered file numbering ascending - self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) - print(expected_last_filename_glob) + file_number = -1 + else: + expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() + expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) + if not glob.glob(expected_last_filename_glob): + self.missing_files.append(expected_last_filename_glob) + else: + file_number = -1 + return file_number if __name__ == '__main__': From ce4ae46c48b297592d3980f5386505c1af89f12a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:50:29 +0100 Subject: [PATCH 1394/2779] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index e51b1ab4e..fb3998b5e 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -31,9 +31,7 @@ Caveats: - This is more complicated than you'd first think as there are so many file naming variations that no code could ever - be universally bulletproof and will require advanced regex tuning to match your use case (this is tuned for the most - common use cases I've found but yours may vary since there are huge file naming variations between people and - environments) + be universally bulletproof and will likely require advanced regex tuning to match your use case and naming convention - Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be. If you are looking for missing MP3 files, then you might be able to check the mp3 tag metadata using programs like @@ -79,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.0' +__version__ = '0.3.1' # pylint: disable=too-many-instance-attributes @@ -195,14 +193,14 @@ def process_directory(self, directory): def check_file(self, filename): log.debug('checking file \'%s\'', filename) - match = self.regex.search(filename) + match = self.regex.search(os.path.basename(filename)) if not match: log.debug('failed to find numeric regex match for file, probably not a sequential file' + \ ', skipping \'%s\'', filename) return # will error out here if you've supplied your own regex without capture brackets # or if you've got pre-captures - let this bubble to user to fix their regex - file_prefix = match.group(1) + file_prefix = os.path.join(os.path.dirname(filename), match.group(1)) file_number = match.group(2) file_suffix = match.group(3) if not isInt(file_number): From 5baaece8aaefa3ed798b190f09e5af3ddab601b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:52:56 +0100 Subject: [PATCH 1395/2779] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index fb3998b5e..4f4382062 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -113,9 +113,9 @@ def add_options(self): '(default: "{}" )'\ .format(self.regex_default)) self.add_opt('-i', '--include', metavar='REGEX', - help='Include only files that match the given case-insensitive regex (eg. ".mp3$")') + help='Include only paths that match the given case-insensitive regex (eg. ".mp3$")') self.add_opt('-e', '--exclude', metavar='REGEX', default=self.exclude_default, - help='Exclude files that match the given case-insensitive regex (default: "{}" )'\ + help='Exclude paths that match the given case-insensitive regex (default: "{}" )'\ .format(self.exclude_default)) self.add_opt('-s', '--fixed-suffix', action='store_true', help='Assume fixed suffixes and infer explicit filenames rather than globs. The reason this ' + \ From 705ebbce495cbeed613b0efc0bd53499c6edfd8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:54:12 +0100 Subject: [PATCH 1396/2779] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 4f4382062..86f3f125b 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -113,7 +113,7 @@ def add_options(self): '(default: "{}" )'\ .format(self.regex_default)) self.add_opt('-i', '--include', metavar='REGEX', - help='Include only paths that match the given case-insensitive regex (eg. ".mp3$")') + help=r"Include only paths that match the given case-insensitive regex (eg. '\.mp3$')") self.add_opt('-e', '--exclude', metavar='REGEX', default=self.exclude_default, help='Exclude paths that match the given case-insensitive regex (default: "{}" )'\ .format(self.exclude_default)) From 00c7ed506752cbff892d33310688e02d8c516c0c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 10:54:58 +0100 Subject: [PATCH 1397/2779] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 86f3f125b..17571602c 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -195,7 +195,7 @@ def check_file(self, filename): log.debug('checking file \'%s\'', filename) match = self.regex.search(os.path.basename(filename)) if not match: - log.debug('failed to find numeric regex match for file, probably not a sequential file' + \ + log.debug('no numeric regex match for file, probably not a sequential file' + \ ', skipping \'%s\'', filename) return # will error out here if you've supplied your own regex without capture brackets From 659dee26ecb4c7ad06c6408a86884744a3a51e4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 1 Aug 2020 11:00:24 +0100 Subject: [PATCH 1398/2779] updated find_missing_files_in_sequence.py --- find_missing_files_in_sequence.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index 17571602c..6d2d8b0db 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -77,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.1' +__version__ = '0.3.2' # pylint: disable=too-many-instance-attributes @@ -92,6 +92,8 @@ def __init__(self): self.regex_default = r'(? Date: Wed, 5 Aug 2020 23:17:01 +0100 Subject: [PATCH 1399/2779] add SQL submodule --- .gitmodules | 3 +++ sql | 1 + 2 files changed, 4 insertions(+) create mode 160000 sql diff --git a/.gitmodules b/.gitmodules index 37537e726..c1f646f43 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,6 @@ path = bash-tools url = https://github.com/harisekhon/bash-tools branch = master +[submodule "sql"] + path = sql + url = https://github.com/HariSekhon/SQL-scripts diff --git a/sql b/sql new file mode 160000 index 000000000..4ffa3e74f --- /dev/null +++ b/sql @@ -0,0 +1 @@ +Subproject commit 4ffa3e74fbc8693b60dd4137d2ee83acaebf7b9a From e6ee82a47ed63e6e33f5ece667130ef0d2f72372 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Aug 2020 23:33:56 +0100 Subject: [PATCH 1400/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 208c2780f..f36c94518 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Hari Sekhon - DevOps Python Tools A few of the Cloud, Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options. -For many more tools see the [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) and [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) repos which contains many Hadoop, NoSQL, Web and infrastructure tools and Nagios plugins. +See also the [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools), [DevOps Perl Tools](https://github.com/harisekhon/devops-perl-tools) and [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) repos which contains hundreds more scripts and programs for Cloud, Big Data, SQL, NoSQL, Web and Linux. Hari Sekhon From a8d1b18ad5c9056da74a30cde5432aa94cc4d18d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 5 Aug 2020 23:37:05 +0100 Subject: [PATCH 1401/2779] updated bootstrap.sh --- setup/bootstrap.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index a77c27e19..6fb41f25c 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -31,7 +31,9 @@ directory="pytools" if [ "$(uname -s)" = Darwin ]; then echo "Bootstrapping Mac" - curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install | ruby + if ! type brew >/dev/null 2>&1; then + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install | ruby + fi elif [ "$(uname -s)" = Linux ]; then echo "Bootstrapping Linux" if type apk >/dev/null 2>&1; then From 22898c8118bc9e56d70fcdb664212815d4903e49 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 03:01:10 +0100 Subject: [PATCH 1402/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2f58c3492..13e94aa3f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2f58c3492e68c7ab1f778197a451fe892827df8f +Subproject commit 13e94aa3fe9b2bf1f3073c031e487e447d76c0ca From f4716e3ba4ef3107f0775d5d493ea4167be51f62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 03:01:11 +0100 Subject: [PATCH 1403/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8a8ed26f1..b37934205 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8a8ed26f1250132446a54f0db627bd14c214bfe4 +Subproject commit b37934205f84f82152144f1c7f294f3223817796 From fb02b24208bf0b848e1b99266b4a78df9d9c66d2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 03:01:11 +0100 Subject: [PATCH 1404/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 4ffa3e74f..f84bb7199 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 4ffa3e74fbc8693b60dd4137d2ee83acaebf7b9a +Subproject commit f84bb7199396b15c5f0cb6055217b030bf338e7c From 9d7dbe21454915080a8ad71b2d3f711a978ec48e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 16:03:50 +0100 Subject: [PATCH 1405/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 13e94aa3f..e658ea4c7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 13e94aa3fe9b2bf1f3073c031e487e447d76c0ca +Subproject commit e658ea4c750414175c4ac0d7985b50ef20011164 From 208549da3825d8193586088692c94ac21df4b131 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 16:03:50 +0100 Subject: [PATCH 1406/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b37934205..7b5e797d2 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b37934205f84f82152144f1c7f294f3223817796 +Subproject commit 7b5e797d2232d6c5cbfc809470dd3c3cde2533ad From be6a07ed054af2cdda4ac075430527efab46f0ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Aug 2020 16:03:50 +0100 Subject: [PATCH 1407/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index f84bb7199..26914f010 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit f84bb7199396b15c5f0cb6055217b030bf338e7c +Subproject commit 26914f0108bb24dec27b08516f8537b73a2aae5e From 0ed46f57b4ca4ceb70b249e6df7556f3f55562b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Aug 2020 12:49:56 +0100 Subject: [PATCH 1408/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e658ea4c7..04685f188 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e658ea4c750414175c4ac0d7985b50ef20011164 +Subproject commit 04685f188aa01e969dd7abbb6ba9e603cf0bae21 From 4add976e16483fb64722e611095d7d942a10cbce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Aug 2020 12:49:56 +0100 Subject: [PATCH 1409/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7b5e797d2..78bd9caa4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7b5e797d2232d6c5cbfc809470dd3c3cde2533ad +Subproject commit 78bd9caa4475e9979a28bcf85d290e9993bc918e From f714f071887bac415d97661ab42e71d2aa40538c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Aug 2020 12:49:56 +0100 Subject: [PATCH 1410/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 26914f010..9736c03eb 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 26914f0108bb24dec27b08516f8537b73a2aae5e +Subproject commit 9736c03eb8b1d8e45a171267065d888b71c8281f From 0014e40f22499e4c623bdaf4bc079427e380563f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 Aug 2020 17:19:01 +0100 Subject: [PATCH 1411/2779] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f36c94518..24c737745 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Hari Sekhon Cloud & Big Data Contractor, United Kingdom -[https://www.linkedin.com/in/harisekhon](https://www.linkedin.com/in/harisekhon) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/harisekhon/) ###### (you're welcome to connect with me on LinkedIn) ##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### From 83f9f1c6a7d3d7bad3bdb6c8aaf8a359da534a92 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 10 Aug 2020 18:18:01 +0100 Subject: [PATCH 1412/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 24c737745..5ded52f24 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Hari Sekhon - DevOps Python Tools -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) From 9038af25597b9275951669bada0f449a31029c1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:43:04 +0100 Subject: [PATCH 1435/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b7a46d5a2..7bf60c2f2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b7a46d5a21cd5150885c232d878dfeb147fe5719 +Subproject commit 7bf60c2f254f327252caeb060521826c60a0c376 From eb3ddb39e198efbe83c30c7525e6a2fdf7ef17a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:43:05 +0100 Subject: [PATCH 1436/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 264e67054..8992cfebe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 264e67054c081ce2c09e0abd7c1f00f7a2155de2 +Subproject commit 8992cfebee776189cd3d8c16edc53a2953846fcf From d57ba74495b6c49012d49a6263679ed687ad1630 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:49:29 +0100 Subject: [PATCH 1437/2779] updated README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 957224c6b..e8130e3b7 100644 --- a/README.md +++ b/README.md @@ -40,15 +40,16 @@ Hari Sekhon - DevOps Python Tools [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) - -[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) +[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) + + +[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) -[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 6cd9b28757c28d4f567e4296f278b020c19ea32f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:54:11 +0100 Subject: [PATCH 1438/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7bf60c2f2..7d19d8c87 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7bf60c2f254f327252caeb060521826c60a0c376 +Subproject commit 7d19d8c876a87f7e87ba852f231abd7304eb89ba From 19268827dcf9ae7465ee41579b8350f2c824c44b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 12:54:11 +0100 Subject: [PATCH 1439/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8992cfebe..f40cc4ae0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8992cfebee776189cd3d8c16edc53a2953846fcf +Subproject commit f40cc4ae05a34bb35197a22f0f32d8e11ad87607 From 18e9d7f0015a7f282f145cd5a5c17982b7968296 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 13:05:33 +0100 Subject: [PATCH 1440/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e8130e3b7..16e5b8849 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,6 @@ Hari Sekhon - DevOps Python Tools [![Docker Build Status](https://img.shields.io/docker/build/harisekhon/pytools?logo=docker)](https://hub.docker.com/r/harisekhon/pytools/builds) [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) -[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) @@ -50,6 +49,7 @@ Hari Sekhon - DevOps Python Tools [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) From 44b1291e95d17624146cab6c8fce47d41fc4a841 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 13:08:25 +0100 Subject: [PATCH 1441/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7d19d8c87..0931d0e8a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7d19d8c876a87f7e87ba852f231abd7304eb89ba +Subproject commit 0931d0e8aa23f507b46c8f89aa6c7006473fda60 From b4642d1ddbf4a860580d46f54291f32e29875d56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 13:08:25 +0100 Subject: [PATCH 1442/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f40cc4ae0..5ccf87c55 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f40cc4ae05a34bb35197a22f0f32d8e11ad87607 +Subproject commit 5ccf87c552d5ff08bfd6a8b20b2a2dfc990206e6 From 2a361ece594cca76f690891179532ea18bb66f42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:15:57 +0100 Subject: [PATCH 1443/2779] updated README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 16e5b8849..440c2dca3 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,11 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) [![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) +[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) +[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) + [![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) @@ -46,11 +51,6 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) -[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) -[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) -[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) -[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) - [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) From dfd2e71edc162b239a036949ee3e0b17eb198384 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:38:46 +0100 Subject: [PATCH 1444/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0931d0e8a..d0d7ac2e5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0931d0e8aa23f507b46c8f89aa6c7006473fda60 +Subproject commit d0d7ac2e54f0bc15988666511a5cde3e6f2c6f19 From 3b0fbf9e9d6ff9a77d1cd90bc34187e4bfc9c0f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:38:46 +0100 Subject: [PATCH 1445/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5ccf87c55..1914a9a16 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5ccf87c552d5ff08bfd6a8b20b2a2dfc990206e6 +Subproject commit 1914a9a16d6958c7d1d27b30c9b3d1170c1879d5 From 3fe7530ad6d9cbd8abd3f6a5f13dd44d4ea0485e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:38:46 +0100 Subject: [PATCH 1446/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 8fc45acbf..ac4bd345b 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 8fc45acbf1346efed706804a042043f98deba060 +Subproject commit ac4bd345bf12263b87a23c9caeaca99ec849bc0d From c1607b90a8a8a1a2dab399af1c1defefe598ef51 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:44:27 +0100 Subject: [PATCH 1447/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 440c2dca3..05d005769 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Hari Sekhon - DevOps Python Tools [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) -[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) +[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis%20CI)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) From 5913af68ea7a5ad939c21ef1f38c51d25e248808 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:52:08 +0100 Subject: [PATCH 1448/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05d005769..8d9e9163b 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Hari Sekhon - DevOps Python Tools [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) -[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) From 62f2e6b9409c6c76f9b62a12198651ace2adcdc9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:53:45 +0100 Subject: [PATCH 1449/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d9e9163b..97bd3dfb3 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Hari Sekhon - DevOps Python Tools [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) From 0072e2befa72b9e0bdfc2b51be5905ad5f935bcb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:55:04 +0100 Subject: [PATCH 1450/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d0d7ac2e5..85c2eb9fe 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d0d7ac2e54f0bc15988666511a5cde3e6f2c6f19 +Subproject commit 85c2eb9fe9bc7c5bc671c088ba27307f84bfc493 From d173e203ef9f56e497b8f043aa11a85aea33e9f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 16:55:04 +0100 Subject: [PATCH 1451/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1914a9a16..d0f5ba5a5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1914a9a16d6958c7d1d27b30c9b3d1170c1879d5 +Subproject commit d0f5ba5a5e75465c86b61e2e02a0a69157b89be4 From f3243cd8c24799d504e0723b8e0cfa3422013be4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Aug 2020 19:15:58 +0100 Subject: [PATCH 1452/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 97bd3dfb3..2c6fe394b 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Hari Sekhon - DevOps Python Tools [![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) -[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) +[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) From 60ed780d90938032be52ed06bc0564760d6ebe71 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:34:58 +0100 Subject: [PATCH 1453/2779] updated README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c6fe394b..1420b8898 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,11 @@ Hari Sekhon - DevOps Python Tools [![Docker](https://img.shields.io/badge/container-Docker-blue?logo=docker)](https://hub.docker.com/r/harisekhon/github/) [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-github?label=DockerHub%20pulls&logo=docker)](https://hub.docker.com/r/harisekhon/github) [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools?logo=docker)](https://hub.docker.com/r/harisekhon/pytools/) +[![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,Dockerfiles&r=HariSekhon,HAProxy-configs) + [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) From 1cdeb0bff9a351d7055f88baa9b194b6803eccf3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:37:52 +0100 Subject: [PATCH 1454/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 85c2eb9fe..6f2d0b5f7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 85c2eb9fe9bc7c5bc671c088ba27307f84bfc493 +Subproject commit 6f2d0b5f785fcdcaff8dfa110fc1017e131971cf From 56be603c68659d0dd249846754c88b4752f5f460 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:37:52 +0100 Subject: [PATCH 1455/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d0f5ba5a5..61ba9c321 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d0f5ba5a5e75465c86b61e2e02a0a69157b89be4 +Subproject commit 61ba9c3219d5392a46bd2ee44d485205da507ca4 From 4cd1521d80b8a41aba51e786def8fd7ea890845b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:37:52 +0100 Subject: [PATCH 1456/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ac4bd345b..5ef8c119f 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ac4bd345bf12263b87a23c9caeaca99ec849bc0d +Subproject commit 5ef8c119f9c2a8158e034c9b84a1489e82989db0 From 52da799e1fbd6c3a58130008a3fd1203b02fe936 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 12:54:48 +0100 Subject: [PATCH 1457/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1420b8898..4c5047983 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Hari Sekhon - DevOps Python Tools [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=shippable)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) +[![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=jfrog)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) From 338e0a29da8cc30e9b5db38f4aca5736a3924b97 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Aug 2020 13:00:05 +0100 Subject: [PATCH 1458/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c5047983..29f51beb4 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Hari Sekhon - DevOps Python Tools [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=jfrog)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) -[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite)](https://buildkite.com/hari-sekhon/devops-python-tools) +[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) From 4bd76109b4770c8de269ce02ae79ca5cbd3d26ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Aug 2020 19:35:31 +0100 Subject: [PATCH 1459/2779] updated README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 29f51beb4..7410ac48f 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,10 @@ Hari Sekhon - DevOps Python Tools [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/network) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) - +[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) -[![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-blue?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) -[![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-blue?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) -[![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-blue?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) +[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) + +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) +[![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) [![GitHub Actions Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/GitHub%20Actions%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22) [![Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac%22) From 4368fcd719288001094f85cd1dc65f892049b560 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Oct 2020 12:42:35 +0100 Subject: [PATCH 1594/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1c4da27cc..d95f2fc67 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1c4da27ccd02c6948d669afbc8dc1e11c261d488 +Subproject commit d95f2fc67f1b0c45a752a0deeb44a257f80a8e26 From ff9de7269f6f48dbdb46f3adff3426c35ac00e57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Oct 2020 12:42:35 +0100 Subject: [PATCH 1595/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d3cde9c87..b47a118cd 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d3cde9c87741dd472ff99d72e8bcfc0e93dc504d +Subproject commit b47a118cdfa3598880d2f8a5df93e4afa9fc21c4 From e450003b73b932fc1dc717c712fe04690b038b97 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Oct 2020 12:42:35 +0100 Subject: [PATCH 1596/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 90847ef41..c83e4a492 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 90847ef411d03bcb3e7f13070d5bedf556148d6b +Subproject commit c83e4a492fbd977d9f6db3841221689d2c49306a From 8c782ad356676c75d241964f46dbd77335a13666 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Oct 2020 12:42:35 +0100 Subject: [PATCH 1597/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 7d2913a4b..6f58cf3a1 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 7d2913a4bf2e65ce6370f754e22638e0b4f9a06d +Subproject commit 6f58cf3a1c329b64419002ecb29bcda768a3e80b From abb0a21a3588e2846e0c07ed91d7efd0cd981eda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 26 Oct 2020 19:54:44 +0000 Subject: [PATCH 1598/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d95f2fc67..a6913bda8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d95f2fc67f1b0c45a752a0deeb44a257f80a8e26 +Subproject commit a6913bda884714f5a489337e32128d3701327b38 From 69cee93beafffe84196b964b6e0917b14d333224 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 26 Oct 2020 19:54:44 +0000 Subject: [PATCH 1599/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6f58cf3a1..03efa398c 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6f58cf3a1c329b64419002ecb29bcda768a3e80b +Subproject commit 03efa398c177cf8d6f2fde6fdace6c686f2f95a9 From 7a340d99e893a44f1e2c6df5cc03a66422775af3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 18:15:10 +0000 Subject: [PATCH 1600/2779] updated requirements.txt --- gcp_cloud_function_sql_export/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcp_cloud_function_sql_export/requirements.txt b/gcp_cloud_function_sql_export/requirements.txt index 47e401b6d..2f7349f4d 100644 --- a/gcp_cloud_function_sql_export/requirements.txt +++ b/gcp_cloud_function_sql_export/requirements.txt @@ -1,3 +1,3 @@ # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python -google-api-python-client -Oauth2client +google-api-python-client==1.12.5 +oauth2client==4.1.3 From 967d20b36bd5c425617b3a3d4a41b55ec959a48b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 20:02:42 +0000 Subject: [PATCH 1601/2779] added gcp_service_account_credential_keys.py --- gcp_service_account_credential_keys.py | 197 +++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100755 gcp_service_account_credential_keys.py diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py new file mode 100755 index 000000000..6435fb3ef --- /dev/null +++ b/gcp_service_account_credential_keys.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-10-29 18:02:14 +0000 (Thu, 29 Oct 2020) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Lists all service account credential keys in a given GCP project + +Excludes built-in system managed keys which are hidden in the Console UI anyway and are not actionable +or in scope for a key policy audit. + + +Output Format: + + + + +You can supply a service account credentials file to authenticate with or just use ADC via: + + gcloud auth application-default login + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from datetime import datetime +import json +import os +import sys +import traceback +from google.oauth2 import service_account +import googleapiclient.discovery +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, log_option, validate_int + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class GcpServiceAccountCredentialKeys(CLI): + + def __init__(self): + # Python 2.x + super(GcpServiceAccountCredentialKeys, self).__init__() + # Python 3.x + # super().__init__() + self.credentials_file = None + self.service = None + self.project = None + self.no_expiry = None + self.expired = None + self.expires_within_days = None + + def add_options(self): + super(GcpServiceAccountCredentialKeys, self).add_options() + self.add_opt('-f', '--credentials-file', metavar='', + default=os.getenv('GOOGLE_CREDENTIALS', \ + os.getenv('GOOGLE_APPLICATION_CREDENTIALS')), + help='Credentials file ($GOOGLE_CREDENTIALS, ' + \ + '$GOOGLE_APPLICATION_CREDENTIALS)') + self.add_opt('-p', '--project-id', metavar='', + help='Google Cloud Project ID ($GOOGLE_PROJECT_ID, or inferred from credentials file') + self.add_opt('-n', '--no-expiry', action='store_true', help='List only non-expiring keys') + self.add_opt('-e', '--expired', action='store_true', help='List only expired keys') + self.add_opt('-d', '--expires-within-days', type=int, help='List only keys that will expire within N days') + + def process_options(self): + super(GcpServiceAccountCredentialKeys, self).process_options() + self.no_args() + project_id = self.get_opt('project_id') + credsfile = self.get_opt('credentials_file') + self.no_expiry = self.get_opt('no_expiry') + self.expired = self.get_opt('expired') + self.expires_within_days = self.get_opt('expires_within_days') + #if not credsfile: + # self.usage('no --credentials-file given and ' + \ + # 'GOOGLE_CREDENTIALS / GOOGLE_APPLICATION_CREDENTIALS environment variables not populated') + if credsfile: + if not os.path.exists(credsfile): + self.usage('credentials file not found: {}'.format(credsfile)) + self.credentials_file = credsfile + log_option('credentials file', self.credentials_file) + if not project_id: + if credsfile: + json_data = json.loads(open(credsfile).read()) + project_id = json_data['project_id'] + else: + self.usage('--project-id not specified and no credentials file given from which to infer') + self.project = project_id + log_option('project', self.project) + if self.expires_within_days is not None: + validate_int(self.expires_within_days, 'expires within days', 0) + if self.no_expiry: + self.usage('--expires-within-days and --no-expiry are mutually exclusive') + if self.expired: + self.usage('--expires-within-days and --expired are mutually exclusive') + if self.no_expiry and self.expired: + self.usage('--expired and --no-expiry are mutually exclusive') + + def run(self): + # defaults to looking for $GOOGLE_APPLICATION_CREDENTIALS or using Application Default Credentials + # from 'gcloud auth application-default login' => ~/.config/gcloud/application_default_credentials.json + credentials = None + if self.credentials_file: + log.debug('loading credentials') + credentials = service_account.Credentials.from_service_account_file( + filename=self.credentials_file, + scopes=['https://www.googleapis.com/auth/cloud-platform'] + ) + + # cache_discovery=False avoids: + # ImportError: file_cache is unavailable when using oauth2client >= 4.0.0 or google-auth + self.service = googleapiclient.discovery.build('iam', 'v1', credentials=credentials, cache_discovery=False) + + for service_account_email in self.get_service_accounts(): + self.list_keys(service_account_email) + + def get_service_accounts(self): + """ Returns a list of service account email addresses """ + + log.debug('getting service accounts') + service_accounts = self.service.projects()\ + .serviceAccounts()\ + .list(name='projects/{}'.format(self.project))\ + .execute() + for account in service_accounts['accounts']: + yield account['email'] + + def list_keys(self, service_account_email): + log.debug("getting keys for service account '%s'", service_account_email) + keys = self.service.projects()\ + .serviceAccounts()\ + .keys()\ + .list(name='projects/-/serviceAccounts/' + service_account_email).execute() + + for key in keys['keys']: + if key['keyType'] == 'SYSTEM_MANAGED': + continue + _id = key['name'].split('/')[-1] + created_date = key['validAfterTime'] + expiry_date = key['validBeforeTime'] + created_datetime = datetime.strptime(created_date, "%Y-%m-%dT%H:%M:%SZ") + expiry_datetime = datetime.strptime(expiry_date, "%Y-%m-%dT%H:%M:%SZ") + age_timedelta = datetime.utcnow() - created_datetime + age_days = int(age_timedelta.total_seconds() / 86400) + expired = False + if expiry_date == '9999-12-31T23:59:59Z': + expires_in_days = 'NEVER' + else: + expires_in_timedelta = expiry_datetime - datetime.utcnow() + expires_in_days = int(expires_in_timedelta.total_seconds() / 86400) + if expires_in_days < 1: + expired = True + if self.no_expiry and (expires_in_days != 'NEVER'): + continue + if self.expired and expired: + continue + if self.expires_within_days is not None and \ + (expires_in_days == 'NEVER' or expires_in_days > self.expires_within_days): + continue + print('{id} {created} {expires} {age:4d} {expires_in:5s} {expired} {service_account}'.format( + id=_id, + created=created_date, + expires=expiry_date, + age=age_days, + expires_in=expires_in_days, + expired=expired, + service_account=service_account_email + )) + + +if __name__ == '__main__': + GcpServiceAccountCredentialKeys().main() From ff83cdd85079221f453535ebc686471be7d3c078 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 20:04:25 +0000 Subject: [PATCH 1602/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 38944cc4d..10ea2c017 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,7 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL scripts to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From 1b62ae4956490739af7d35089b19ee7ea8e52341 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 20:05:03 +0000 Subject: [PATCH 1603/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 10ea2c017..6f339e6dd 100644 --- a/README.md +++ b/README.md @@ -170,8 +170,8 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL scripts to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` From a4492d393420c6b7d0e8c7dd4d2ffc438a79cbf9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 29 Oct 2020 20:06:18 +0000 Subject: [PATCH 1604/2779] updated gcp_service_account_credential_keys.py --- gcp_service_account_credential_keys.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py index 6435fb3ef..bd4460f35 100755 --- a/gcp_service_account_credential_keys.py +++ b/gcp_service_account_credential_keys.py @@ -32,6 +32,11 @@ gcloud auth application-default login + +See Also - similar scripts in the DevOps Bash tools repo: + + https://github.com/HariSekhon/DevOps-Bash-tools/ + """ from __future__ import absolute_import From 39d1ba5b1a6f9a2b395fe3e941f988a49022e994 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 2 Nov 2020 10:29:14 +0000 Subject: [PATCH 1605/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f339e6dd..45c41ab21 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL scripts to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From cf441dcac8f0e24964b35ae81bfe906df7f8217d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 20 Nov 2020 18:53:08 +0000 Subject: [PATCH 1606/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 45c41ab21..de5c8ebd5 100644 --- a/README.md +++ b/README.md @@ -438,7 +438,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 450+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 500+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies * [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From 9c6db0c6c51a2e39ad0be12d5cb5fe6086a966ca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 09:36:55 +0000 Subject: [PATCH 1607/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a6913bda8..42b187dd8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a6913bda884714f5a489337e32128d3701327b38 +Subproject commit 42b187dd8150fcdb872d5b07191129202e6bbe2d From 0ebfc991e793d3cef90f1b3b1ee96415e0ef23a3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 09:36:56 +0000 Subject: [PATCH 1608/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b47a118cd..3338b29be 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b47a118cdfa3598880d2f8a5df93e4afa9fc21c4 +Subproject commit 3338b29be1de01b2f72677e868eccc19bdec0bbc From c4e8f98c64b85bc5b2734753926bc1b66ffb7cf8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 09:36:56 +0000 Subject: [PATCH 1609/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index c83e4a492..8039c0aba 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit c83e4a492fbd977d9f6db3841221689d2c49306a +Subproject commit 8039c0abae11bc4595dea5af9c9a7e237015cc94 From f066926deee69c073ff3c93a011cf7b24caf7459 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 09:36:56 +0000 Subject: [PATCH 1610/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 03efa398c..9a2a821bd 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 03efa398c177cf8d6f2fde6fdace6c686f2f95a9 +Subproject commit 9a2a821bd85d3bae4524f16164cae03039c1a4e3 From 0754cfa6761a78b7bd56ec77c58116655ff96e83 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 14:26:32 +0000 Subject: [PATCH 1611/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 42b187dd8..a03cc5415 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 42b187dd8150fcdb872d5b07191129202e6bbe2d +Subproject commit a03cc54157363a0212f220eedbdce5714bd66f1e From c0206ba5e449c26c836ee04cdd4f0e9e7d2f557f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Nov 2020 14:26:33 +0000 Subject: [PATCH 1612/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3338b29be..6e250073e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3338b29be1de01b2f72677e868eccc19bdec0bbc +Subproject commit 6e250073e5efe1110cdb43630b1778e5c8db4776 From 297e6d6d6e2e3ae50fbffcfec84b74fd473b30d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Dec 2020 23:12:09 +0000 Subject: [PATCH 1613/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index de5c8ebd5..f76dfe6f0 100644 --- a/README.md +++ b/README.md @@ -438,7 +438,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 500+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 550+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies * [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From 963a7ed903c1c0ae7cf6eab42d00287430cc8024 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Dec 2020 23:52:19 +0000 Subject: [PATCH 1614/2779] updated shippable.yml --- shippable.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shippable.yml b/shippable.yml index 0c19cb053..71c6c3c76 100644 --- a/shippable.yml +++ b/shippable.yml @@ -32,6 +32,9 @@ build: # devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed - rm -fv /etc/apt/sources.list.d/cassandra.sources.list* - rm -fv /etc/apt/sources.list.d/yarn.list* + # Basho repo is giving a '402 payment required' error + # https://github.com/Shippable/support/issues/5172 + - rm -fv /etc/apt/sources.list.d/basho_riak.list #- shippable_retry make - setup/ci_bootstrap.sh - make init From 63d073f5cb280ea07743e1a390b10c282cf3a79b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 4 Dec 2020 23:20:21 +0000 Subject: [PATCH 1615/2779] updated test_anonymize.sh --- tests/test_anonymize.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 53b219274..713b844ea 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -546,6 +546,8 @@ fi # this gives the number of elements and prevents testing the last element(s) if commenting something out in the middle #for (( i = 0 ; i < ${#src[@]} ; i++ )); do run_tests(){ + # expands to the list of indicies in the array, starting at zero - this is easier to work with that ${#src} which is a total + # that is off by one for index usage and doesn't support sparse arrays for any missing/disabled test indicies test_numbers="${*:-${!src[*]}}" for i in $test_numbers; do [ -n "${src[$i]:-}" ] || { echo "code error: src[$i] not defined"; exit 1; } From e98fef5b5819c89c203b05335bf3c01681db62b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 5 Dec 2020 21:11:30 +0000 Subject: [PATCH 1616/2779] updated .travis.yml --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf0870f49..fcdaf693a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,10 @@ python: - "pypy" # currently Python 2.7.13, PyPy 7.1.1 - "pypy3" # currently Python 3.6.1, PyPy 7.1.1-beta0 +# https://docs.travis-ci.com/user/reference/osx/ +# macOS 10.15.7 - otherwise defaults to Mac macOS 10.13 with xcode9.4 otherwise - and HomeBrew update takes 50 minutes until the build times out :-/ +osx_image: xcode12.2 + matrix: fast_finish: true include: @@ -71,7 +75,6 @@ matrix: language: python python: "pypy3" - # allow_failures: # - python: "3.4" # - python: "3.5" @@ -81,8 +84,6 @@ matrix: # - python: "pypy" # - python: "pypy3" -#dist: trusty - sudo: required env: From f82255dc507bdfca6268197c2f5f18ac6648f046 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 5 Dec 2020 22:55:32 +0000 Subject: [PATCH 1617/2779] updated .travis.yml --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcdaf693a..fcd6630b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,10 +30,6 @@ python: - "pypy" # currently Python 2.7.13, PyPy 7.1.1 - "pypy3" # currently Python 3.6.1, PyPy 7.1.1-beta0 -# https://docs.travis-ci.com/user/reference/osx/ -# macOS 10.15.7 - otherwise defaults to Mac macOS 10.13 with xcode9.4 otherwise - and HomeBrew update takes 50 minutes until the build times out :-/ -osx_image: xcode12.2 - matrix: fast_finish: true include: @@ -44,6 +40,9 @@ matrix: - os: osx language: generic # workaround since Mac doesn't have Python support yet, so install to system Python + # https://docs.travis-ci.com/user/reference/osx/ + # macOS 10.15.7 - otherwise defaults to Mac macOS 10.13 with xcode9.4 otherwise - and HomeBrew update takes 50 minutes until the build times out :-/ + osx_image: xcode12.2 - os: linux language: python From 20428ff8bdc2943bca54875dcdbf89e71a0c00ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 7 Dec 2020 23:27:27 +0000 Subject: [PATCH 1618/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f76dfe6f0..b4a8b8480 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Hari Sekhon - DevOps Python Tools [![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/centos-github?label=DockerHub%20pulls&logo=docker)](https://hub.docker.com/r/harisekhon/github) [![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools?logo=docker)](https://hub.docker.com/r/harisekhon/pytools/) [![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,Dockerfiles&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,HAProxy-configs&r=HariSekhon,SQL-scripts) +[![StarCharts](https://img.shields.io/badge/Star-Charts-blue?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STARCHARTS.md) From eb7242dbfbd8b2e78009c13148763081325c1d94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Jan 2021 18:09:46 +0000 Subject: [PATCH 1674/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9c47e9ece..46da98469 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9c47e9ece389cf07c29d414bfb652059cb93b2c6 +Subproject commit 46da98469a78ac68e835b4f72088dcb6ad8bd11c From 589cb2303b6262ee3d2f10c554b458b2bb86d49d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Jan 2021 18:09:46 +0000 Subject: [PATCH 1675/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f5ceb0e08..204fe01f4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f5ceb0e0840ba35170277e3d978eae2187063be5 +Subproject commit 204fe01f4ed7c0bd5f8a83fc972ae4589b43bdd4 From afc44f30d4663f7c323aec108e1dafe64ea38203 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Jan 2021 18:50:02 +0000 Subject: [PATCH 1676/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 46da98469..01b9badbb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 46da98469a78ac68e835b4f72088dcb6ad8bd11c +Subproject commit 01b9badbb5c237f07b2dce1108f606be4a166dfa From 3de9f649db081b79955d8c79f8b85b7b8252e5f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Jan 2021 18:50:02 +0000 Subject: [PATCH 1677/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 204fe01f4..efb7245b3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 204fe01f4ed7c0bd5f8a83fc972ae4589b43bdd4 +Subproject commit efb7245b32876d494e9c02ac718277a0e4c0c0b5 From cc345093f15e758215b46a1f8c9a1c8e0093d130 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 18 Jan 2021 18:50:03 +0000 Subject: [PATCH 1678/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c42667ad8..5ddab4747 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c42667ad8423aeb06d8222c76cd4d120ad30f4ee +Subproject commit 5ddab474791ca70ff9870373584dc9c65c017ba8 From 1c60303d95009247bb0f2024cd5e5edf080ef01a Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 21 Jan 2021 06:39:28 +0000 Subject: [PATCH 1679/2779] fix: requirements.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-PYYAML-590151 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f2ceaaa9f..d7fd0b720 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,3 +56,4 @@ thriftpy==0.3.9 toml==0.10.0 xmltodict==0.10.2 yamllint==1.15.0 +pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability From 62da28abc3344cac236dce723a837d9859e13b7b Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 2 Feb 2021 06:40:04 +0000 Subject: [PATCH 1680/2779] fix: requirements.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-JINJA2-1012994 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f2ceaaa9f..dadd75a0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ GitPython==2.1.15 happybase==1.0.0 humanize==0.5.1 impyla==0.16.0 -Jinja2==2.10.1 +Jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 #MarkupSafe==0.23 From 366bf335996364f104706729487103c977addb6b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 15:57:43 +0000 Subject: [PATCH 1681/2779] renamed .teamcity.vcs.auth.json to .teamcity.vcs.oauth.json --- .teamcity.vcs.auth.json => .teamcity.vcs.oauth.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.auth.json => .teamcity.vcs.oauth.json (100%) diff --git a/.teamcity.vcs.auth.json b/.teamcity.vcs.oauth.json similarity index 100% rename from .teamcity.vcs.auth.json rename to .teamcity.vcs.oauth.json From 32f98ca58eae04ebcbe2ecb74509059e47a901a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 15:59:50 +0000 Subject: [PATCH 1682/2779] added .teamcity.vcs.ssh.json --- .teamcity.vcs.ssh.json | 65 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .teamcity.vcs.ssh.json diff --git a/.teamcity.vcs.ssh.json b/.teamcity.vcs.ssh.json new file mode 100644 index 000000000..1b780675a --- /dev/null +++ b/.teamcity.vcs.ssh.json @@ -0,0 +1,65 @@ +{ + "id": "TeamCity", + "name": "TeamCity", + "vcsName": "jetbrains.git", + "href": "/app/rest/vcs-roots/id:TeamCity", + "project": { + "id": "_Root", + "name": "", + "description": "Contains all other projects", + "href": "/app/rest/projects/id:_Root", + "webUrl": "http://localhost:8111/project.html?projectId=_Root" + }, + "properties": { + "count": 11, + "property": [ + { + "name": "agentCleanFilesPolicy", + "value": "ALL_UNTRACKED" + }, + { + "name": "agentCleanPolicy", + "value": "ON_BRANCH_CHANGE" + }, + { + "name": "authMethod", + "value": "TEAMCITY_SSH_KEY" + }, + { + "name": "branch", + "value": "refs/heads/master" + }, + { + "name": "ignoreKnownHosts", + "value": "true" + }, + { + "name": "submoduleCheckout", + "value": "CHECKOUT" + }, + { + "name": "teamcitySshKey", + "value": "VCS SSH Key" + }, + { + "name": "url", + "value": "github.com:HariSekhon/TeamCity-CI" + }, + { + "name": "useAlternates", + "value": "true" + }, + { + "name": "username", + "value": "git" + }, + { + "name": "usernameStyle", + "value": "USERID" + } + ] + }, + "vcsRootInstances": { + "href": "/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)" + } +} From 6f0fa2482bf91791702f266b92df47732822b315 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:24:43 +0000 Subject: [PATCH 1683/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 01b9badbb..9410e2913 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 01b9badbb5c237f07b2dce1108f606be4a166dfa +Subproject commit 9410e29130a0567a78d0f6e24a33b13f950a32d0 From 63a6fc04cdd4233d374fbdb9ac750bd347eb48db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:24:43 +0000 Subject: [PATCH 1684/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index efb7245b3..cefc87fbb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit efb7245b32876d494e9c02ac718277a0e4c0c0b5 +Subproject commit cefc87fbb8278156ffaec1c2930cdba66e55c4ad From 7c98f118f9d9db843a570c9797cec3f0a7a0c52b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:24:44 +0000 Subject: [PATCH 1685/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ad0d35a9a..2f8bd447f 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ad0d35a9a2417a8cbdef65c0f9cbbe023394d408 +Subproject commit 2f8bd447f236a4875a161f41ac30684b227acdf6 From 40fb56c4241b612854f9ecd8c1bfff0f4a40b80e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:24:44 +0000 Subject: [PATCH 1686/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 5ddab4747..f6dd9a3d0 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 5ddab474791ca70ff9870373584dc9c65c017ba8 +Subproject commit f6dd9a3d0b5403949b01f0a2f13f0d3c4c894b02 From 468976a7fdd6e5f7a416ac3666831d886e540330 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:37:03 +0000 Subject: [PATCH 1687/2779] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index 29718e8b3..691e04a2f 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -39,7 +39,7 @@ retry(){ "$@" && break; echo - if [ $count -ge $max_tries ]; then + if [ "$count" -ge "$max_tries" ]; then echo "$count tries failed, aborting..." exit 1 fi From f0f8a293ee18d04f218b3d3a0802ca5606a51192 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:42:06 +0000 Subject: [PATCH 1688/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9410e2913..3bf5112f4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9410e29130a0567a78d0f6e24a33b13f950a32d0 +Subproject commit 3bf5112f4efa8664f64dc37cedb0c3c130d90356 From 9edc5afa50931e8e9d1ed5550ac327361ba4c256 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:42:06 +0000 Subject: [PATCH 1689/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index cefc87fbb..3d2bd6438 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit cefc87fbb8278156ffaec1c2930cdba66e55c4ad +Subproject commit 3d2bd6438b0cf6fcc1ea5c47ff0b7140edd2fef1 From 02bbb77a5b9bf91baa742a6881ff6bf2f42b25f6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 8 Feb 2021 16:42:07 +0000 Subject: [PATCH 1690/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index f6dd9a3d0..58dc8e0c4 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit f6dd9a3d0b5403949b01f0a2f13f0d3c4c894b02 +Subproject commit 58dc8e0c4ac85128c8f0059b6c762d4e8c163de4 From ec394d767c75f02c1bf440ece406f285a93c0ec1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 22:51:35 +0000 Subject: [PATCH 1691/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b22c5486b..69316433b 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Hari Sekhon - DevOps Python Tools [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) -[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) +[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) [![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) From 04c9a928c25af511e280ea0b778480d65057066f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 22:56:20 +0000 Subject: [PATCH 1692/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3bf5112f4..0298b6224 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3bf5112f4efa8664f64dc37cedb0c3c130d90356 +Subproject commit 0298b6224d23a0648df7001d945960c9462bc4c9 From e1c534ccc8992b87a056a46e93e50e38dc9a5b57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 22:56:20 +0000 Subject: [PATCH 1693/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3d2bd6438..539fd5b4d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3d2bd6438b0cf6fcc1ea5c47ff0b7140edd2fef1 +Subproject commit 539fd5b4de2a77f8f36bb19e9ee4b969dbd8d384 From 83b090d45aeb699ea8a0727c75fd3353df5ae127 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 22:56:20 +0000 Subject: [PATCH 1694/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 58dc8e0c4..d57e9cac4 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 58dc8e0c4ac85128c8f0059b6c762d4e8c163de4 +Subproject commit d57e9cac4960159be9887113a4c2f30b2c9bade4 From fe9ca366b127af6b42cd85b279fd18668e5d457b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 23:03:49 +0000 Subject: [PATCH 1695/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0298b6224..23cecb014 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0298b6224d23a0648df7001d945960c9462bc4c9 +Subproject commit 23cecb014fbe8d5b103d01e9fa487687c95d7859 From 8b78c3299d3f2513cc43599d15ba1b91e79e42b5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 23 Feb 2021 23:04:38 +0000 Subject: [PATCH 1696/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 539fd5b4d..f6eb8fa48 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 539fd5b4de2a77f8f36bb19e9ee4b969dbd8d384 +Subproject commit f6eb8fa48f06cd42b692bf1a61c5b84915bdd015 From 2e460237e87eca44649762355c8a3ee5c0fea8e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Mar 2021 18:18:16 +0000 Subject: [PATCH 1697/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 23cecb014..c28241777 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 23cecb014fbe8d5b103d01e9fa487687c95d7859 +Subproject commit c28241777474d0291f8d5422a55a23b3a92e2b49 From ebdddf0cd33fde7cec8fb8d95b660350d19417b6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Mar 2021 18:18:17 +0000 Subject: [PATCH 1698/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index f6eb8fa48..a20b1b987 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit f6eb8fa48f06cd42b692bf1a61c5b84915bdd015 +Subproject commit a20b1b987cae04ec098b6b27969f185f90307430 From da16282b46f208bfbde5ad9a3f90eb9eb94a1da4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Mar 2021 18:18:17 +0000 Subject: [PATCH 1699/2779] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 2f8bd447f..9cefa28a2 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 2f8bd447f236a4875a161f41ac30684b227acdf6 +Subproject commit 9cefa28a28962e1b6f1bb935c2a405b62bcba5d1 From 17e4a0709e2a057baa16ea782bb585a1285e3aea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Mar 2021 18:18:17 +0000 Subject: [PATCH 1700/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d57e9cac4..dc906b935 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d57e9cac4960159be9887113a4c2f30b2c9bade4 +Subproject commit dc906b9356ed7bd8d8c8c6c3079cd7b8acf7ece7 From c18f5def8b103dba54629372989bace7668427ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 19 Mar 2021 18:44:24 +0000 Subject: [PATCH 1701/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 69316433b..b8570d3ab 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,10 @@ Hari Sekhon - DevOps Python Tools [![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=jfrog)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) -[![buddy pipeline](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990/badge.svg?token=7f63afa3c423a65e6e39a79be0386959e98c4105ea1e20f7f8b05d6d6b587038 "buddy pipeline")](https://app.buddy.works/harisekhon/devops-python-tools/pipelines/pipeline/246990) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) +[![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) From 592385a88f2d17e01e1a2505a39a53f2c045e3fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 12:51:50 +0100 Subject: [PATCH 1702/2779] updated Jenkinsfile --- Jenkinsfile | 150 +++++++++++++++++++--------------------------------- 1 file changed, 53 insertions(+), 97 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c9fbad7d3..c95f1d1e7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,120 +14,76 @@ // // ========================================================================== // -// J e n k i n s P i p e l i n e +// J e n k i n s P i p e l i n e // ========================================================================== // +// Epic Jenkinsfile template: +// +// https://github.com/HariSekhon/Templates/blob/master/Jenkinsfile + +// Official Documentation: +// // https://jenkins.io/doc/book/pipeline/syntax/ +// +// https://www.jenkins.io/doc/pipeline/steps/ +// +// https://www.jenkins.io/doc/pipeline/steps/workflow-basic-steps/ pipeline { - // run pipeline any agent - agent any - // can't do this when running jenkins in docker itself, gets '.../script.sh: docker: not found' -// agent { -// docker { -// image 'ubuntu:18.04' -// args '-v $HOME/.m2:/root/.m2 -v $HOME/.cache/pip:/root/.cache/pip -v $HOME/.cpanm:/root/.cpanm -v $HOME/.sbt:/root/.sbt -v $HOME/.ivy2:/root/.ivy2 -v $HOME/.gradle:/root/.gradle' -// } -// } + // to run on Docker or Kubernetes, see the master Jenkinsfile template listed at the top + agent any - // need to specify at least one env var if enabling - //environment { - // DEBUG = '1' - //} + options { + timestamps() - options { - // put timestamps in console logs - timestamps() + timeout(time: 2, unit: 'HOURS') + } - // timeout entire pipeline after 4 hours - timeout(time: 4, unit: 'HOURS') + triggers { + cron('H 10 * * 1-5') + pollSCM('H/2 * * * *') + } - //retry entire pipeline 3 times - //retry(3) + stages { + stage ('Checkout') { + steps { + checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]]) + } } - triggers { - cron('H 10 * * 1-5') - pollSCM('H/2 * * * *') - } - - stages { - stage ('Checkout') { - steps { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]]) - } + stage('Build') { + steps { + echo "Running ${env.JOB_NAME} Build ${env.BUILD_ID} on ${env.JENKINS_URL}" + echo 'Building...' + timeout(time: 10, unit: 'MINUTES') { + retry(3) { +// sh 'apt update -q' +// sh 'apt install -qy make' +// sh 'make init' + sh """ + setup/ci_bootstrap.sh && + make init + """ + } } - - stage('Build') { - steps { - echo "Running ${env.BUILD_ID} on ${env.JENKINS_URL}" - echo 'Building...' - timeout(time: 10, unit: 'MINUTES') { - retry(3) { -// sh 'apt update -q' -// sh 'apt install -qy make' -// sh 'make init' - sh """ - setup/ci_bootstrap.sh && - make init - """ - } - } - timeout(time: 180, unit: 'MINUTES') { - sh 'make ci' - } - } + timeout(time: 180, unit: 'MINUTES') { + sh 'make ci' } - - stage('Test') { - options { - retry(2) - } - steps { - echo 'Testing...' - timeout(time: 120, unit: 'MINUTES') { - sh 'make test' - } - } - } - -// stage('Human gate') { -// steps { -// input "Proceed to deployment?" -// } -// } -// -// stage('Deployment') { -// steps { -// echo 'Deploying...' -// echo 'Nothing to deploy' -// } -// } - + } } - post { - always { - echo 'Always' - //deleteDir() // clean up workspace - // collect JUnit reports for Jenkins UI - //junit 'build/reports/**/*.xml' - // collect artifacts to Jenkins for analysis - //archiveArtifacts artifacts: 'build/libs/**/*.jar', fingerprint: true - } - success { - echo 'SUCCESS!' - } - failure { - echo 'FAILURE!' - } - unstable { - echo 'UNSTABLE!' - } - changed { - echo 'Pipeline state change! (success vs failure)' + stage('Test') { + options { + retry(2) + } + steps { + echo 'Testing...' + timeout(time: 120, unit: 'MINUTES') { + sh 'make test' } + } } + } } From 261f5e99c6a9d71a9b548534412fad3de6a9bcf0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 12:53:49 +0100 Subject: [PATCH 1703/2779] updated Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c95f1d1e7..7739f92e3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ // // ========================================================================== // -// J e n k i n s P i p e l i n e +// J e n k i n s P i p e l i n e // ========================================================================== // // Epic Jenkinsfile template: From b361f83215c5128d16283a412f9cd610038b4a21 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 13:47:29 +0100 Subject: [PATCH 1704/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c28241777..c13327c6e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c28241777474d0291f8d5422a55a23b3a92e2b49 +Subproject commit c13327c6ef2c1b4354fd49f559b42b7094d0a80b From 51e6794f10a7b739c0261373231a140cb914d979 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 13:47:29 +0100 Subject: [PATCH 1705/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a20b1b987..41beb39b8 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a20b1b987cae04ec098b6b27969f185f90307430 +Subproject commit 41beb39b8e17734cfc4598e11f5da3043a15f829 From adbb8dc480f3531d351882f89c89405c12422992 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 13:47:29 +0100 Subject: [PATCH 1706/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 9cefa28a2..80e2004a1 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 9cefa28a28962e1b6f1bb935c2a405b62bcba5d1 +Subproject commit 80e2004a17865c23992f3ad8d5bae3d55e60dc46 From 32d27d5f4014f07c632c52842eff1a9129a97deb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Apr 2021 13:47:29 +0100 Subject: [PATCH 1707/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index dc906b935..25324f745 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit dc906b9356ed7bd8d8c8c6c3079cd7b8acf7ece7 +Subproject commit 25324f745c7866a0d2990e60bec24342655e1185 From 5230833e5b877aebb3b49214f166b5fc7848233f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:18:09 +0100 Subject: [PATCH 1708/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c13327c6e..89e5adffb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c13327c6ef2c1b4354fd49f559b42b7094d0a80b +Subproject commit 89e5adffb393097a8d32ffece93bb299ac56e46b From 088780143cc0bbd61d284448bfa738af2c899307 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:18:09 +0100 Subject: [PATCH 1709/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 41beb39b8..28f571cce 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 41beb39b8e17734cfc4598e11f5da3043a15f829 +Subproject commit 28f571cceaabc4e0025cd2513b70d8f5db52e1a6 From df07261681abdf0c112b20394c985bf0abf9281a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:44:40 +0100 Subject: [PATCH 1710/2779] added codeship.yml --- codeship.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 codeship.yml diff --git a/codeship.yml b/codeship.yml new file mode 100644 index 000000000..8b119709c --- /dev/null +++ b/codeship.yml @@ -0,0 +1,30 @@ +# +# Author: Hari Sekhon +# Date: 2021-04-12 18:33:44 +0100 (Mon, 12 Apr 2021) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this + +# https://github.com/painless-software/codeship-yaml + +--- +install: + - sudo rm -fv /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list + - sudo rm -fv /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list + - make +#before_script: +# - somecommand +script: + - make test +#after_success: +# - echo "Now we can deploy" From f105b83090ef89baa4b9680caf6bc91493bd7e42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:54:31 +0100 Subject: [PATCH 1711/2779] updated codeship.yml --- codeship.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/codeship.yml b/codeship.yml index 8b119709c..53efd9424 100644 --- a/codeship.yml +++ b/codeship.yml @@ -13,14 +13,40 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e S h i p +# ============================================================================ # + # 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this # https://github.com/painless-software/codeship-yaml +# Requires setting up the CodeShip commands like so: +# +# pip install codeship-yaml +# codeship-yaml +# +# or seaparately in sections: +# +# Project Settings > Test Settings > Setup Commands: +# +# pip install codeship-yaml +# codeship-yaml install +# +# Project Settings > Test Settings > Test Commands: +# +# codeship-yaml before_script script +# +# Project Settings > Deployment > (branch name) +# +# codeship-yaml after_success + --- install: + # these cause package installation breakages due to GPG or 403 errors, old addresses etc. - sudo rm -fv /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list - sudo rm -fv /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list + - sudo rm -fv /etc/apt/sources.list.d/www_apache_org_dist_cassandra_debian.list - make #before_script: # - somecommand From c8ba8395b932e6be810f750e1509e6d77b5de0f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:57:00 +0100 Subject: [PATCH 1712/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 89e5adffb..bc2d191fa 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 89e5adffb393097a8d32ffece93bb299ac56e46b +Subproject commit bc2d191fad8b63df60944f902da10f3cb4d0eaea From 4ce4f7ad732c8a12f78a2a32bf7b11573e1e187c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 12 Apr 2021 18:57:01 +0100 Subject: [PATCH 1713/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 28f571cce..65631842d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 28f571cceaabc4e0025cd2513b70d8f5db52e1a6 +Subproject commit 65631842daa333da9e8f5ca4ea26f30d380e53b6 From 1b373a178f0402fc6e9d9df2c44603db0f724389 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 17:35:29 +0100 Subject: [PATCH 1714/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bc2d191fa..b1363ed43 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bc2d191fad8b63df60944f902da10f3cb4d0eaea +Subproject commit b1363ed437868c8eed63d5aa628c1fee8cb676c0 From 222035289f999345b4f3009ee561618dcff8e894 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 17:35:30 +0100 Subject: [PATCH 1715/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 25324f745..a9ebed85a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 25324f745c7866a0d2990e60bec24342655e1185 +Subproject commit a9ebed85a6753ef025b8d1c49fdb6a466bb66727 From 0026bde0659f2affa328ee345844c08697e400f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 17:41:00 +0100 Subject: [PATCH 1716/2779] added dynamic targets for pyc and pyo creation --- Makefile | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d939369eb..9960a1d0b 100755 --- a/Makefile +++ b/Makefile @@ -74,12 +74,24 @@ build: init init: git submodule update --init --recursive +# Magic to build the dependencies for only given program(s) +%.pyc: + @# this utility script supports taking .pyc or .pyo names and still does the right thing + bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + python -m py_compile `echo $@ | sed 's/\.pyc$$/.py/'` +%.pyo: + bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + python -O -m py_compile `echo $@ | sed 's/\.pyo$$/.py/'` + +.PHONY: pylib +pylib: + @$(MAKE) python-version + cd pylib && $(MAKE) + .PHONY: python -python: +python: pylib # defer via external sub-call, otherwise will result in error like # make: *** No rule to make target 'python-version', needed by 'build'. Stop. - @$(MAKE) python-version - cd pylib && $(MAKE) @# don't pull parquet tools in to docker image by default, will bloat it @# can fetch separately by running 'make parquet-tools' if you really want to @if [ -f /.dockerenv -o -n "$(SKIP_PARQUET)" ]; then \ From 8a1347736fcdad4b0add0d9f8841eddaf08723e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 17:49:21 +0100 Subject: [PATCH 1717/2779] updated Makefile --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9960a1d0b..b835f8699 100755 --- a/Makefile +++ b/Makefile @@ -75,11 +75,14 @@ init: git submodule update --init --recursive # Magic to build the dependencies for only given program(s) -%.pyc: +# +# dependency of same % stem prefix checks for a matching .py file to consider it a valid target +# +%.pyc: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile `echo $@ | sed 's/\.pyc$$/.py/'` -%.pyo: +%.pyo: %.py bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile `echo $@ | sed 's/\.pyo$$/.py/'` From 8207890a84d2f22f4881850579fc7e3a3e2ee697 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 18:09:04 +0100 Subject: [PATCH 1718/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b1363ed43..7b69c6157 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b1363ed437868c8eed63d5aa628c1fee8cb676c0 +Subproject commit 7b69c61571a3a27b786884999fd0937a8150cc78 From 2c6be8587928a68388f0e181ae26df1a2e8299ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 18:09:42 +0100 Subject: [PATCH 1719/2779] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b835f8699..e8ef4c60d 100755 --- a/Makefile +++ b/Makefile @@ -80,10 +80,10 @@ init: # %.pyc: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing - bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile `echo $@ | sed 's/\.pyc$$/.py/'` %.pyo: %.py - bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile `echo $@ | sed 's/\.pyo$$/.py/'` .PHONY: pylib From 393ec22b215d81f11c8a3a8f7e728b97e20fc5c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 18:09:51 +0100 Subject: [PATCH 1720/2779] updated README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index b8570d3ab..18daac3e3 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,11 @@ cd pytools make ``` +To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension instead of `.py`: +``` +make anonymize.pyc +``` + Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/devops-python-tools#detailed-build-instructions) further down for more information. Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/harisekhon/devops-python-tools#jython-for-hadoop-utils) for details. From 2d8338292a0c9a9b133bfa4a061330d0fb88ff7f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 7 May 2021 18:13:40 +0100 Subject: [PATCH 1721/2779] updated Makefile --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e8ef4c60d..ac40a686d 100755 --- a/Makefile +++ b/Makefile @@ -79,12 +79,16 @@ init: # dependency of same % stem prefix checks for a matching .py file to consider it a valid target # %.pyc: %.py - @# this utility script supports taking .pyc or .pyo names and still does the right thing + @# this utility script supports taking .pyc or .pyo names and still does the right thing, @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ - python -m py_compile `echo $@ | sed 's/\.pyc$$/.py/'` + python -m py_compile $< && \ + echo && \ + echo Generated $@ %.pyo: %.py @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ - python -O -m py_compile `echo $@ | sed 's/\.pyo$$/.py/'` + python -O -m py_compile $< && \ + echo && \ + echo Generated $@ .PHONY: pylib pylib: From 53b26cae33944a01fe81c84a62d637b9d1037993 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 8 May 2021 13:11:25 +0100 Subject: [PATCH 1722/2779] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ac40a686d..abe77d452 100755 --- a/Makefile +++ b/Makefile @@ -78,13 +78,13 @@ init: # # dependency of same % stem prefix checks for a matching .py file to consider it a valid target # -%.pyc: %.py +%.pyc:: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing, @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ echo Generated $@ -%.pyo: %.py +%.pyo:: %.py @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile $< && \ echo && \ From 9e3b722b55ec604d249b109c980312b4fb4b49fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 10:44:36 +0100 Subject: [PATCH 1723/2779] added selenium_test.py --- selenium_test.py | 145 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100755 selenium_test.py diff --git a/selenium_test.py b/selenium_test.py new file mode 100755 index 000000000..3a4118e43 --- /dev/null +++ b/selenium_test.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-12 09:55:01 +0100 (Wed, 12 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Tests a Selenium Hub / Selenoid using the given capability eg. FIREFOX, CHROME +against a given URL and content (defaults to google.com) + +Example: + + ./selenium_test.py --host [options] + + ./selenium_test.py --host selenium-hub FIREFOX CHROME + + ./selenium_test.py --host selenium-hub FIREFOX CHROME --url google.com --content google + ./selenium_test.py --host selenium-hub FIREFOX CHROME --url google.com --regex 'goog.*' + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import re +import sys +import time +import traceback +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from harisekhon.utils import validate_host, validate_port, validate_url, validate_regex + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class SeleniumTest(CLI): + + def __init__(self): + # Python 2.x + super(SeleniumTest, self).__init__() + # Python 3.x + # super().__init__() + self.host = None + self.port = None + self.protocol = 'http' + self.name = 'Selenium Hub' + self.default_port = 80 + self.path = 'wd/hub' + self.url_default = 'http://google.com' + self.url = self.url_default + self.expected_content = None + self.expected_regex = None + self.timeout_default = 600 + self.verbose_default = 2 + + def add_options(self): + super(SeleniumTest, self).add_options() + self.add_hostoption(name='Selenium Hub', default_port=4444) + self.add_opt('-u', '--url', default=self.url_default, + help='URL to use for the test (default: {})'.format(self.url_default)) + self.add_opt('-c', '--content', help='URL content to expect') + self.add_opt('-r', '--regex', help='URL content to expect') + self.add_opt('-S', '--ssl', action='store_true', help='Use SSL to connect to Selenium Hub') + + def process_options(self): + super(SeleniumTest, self).process_options() + self.host = self.get_opt('host') + self.port = self.get_opt('port') + self.url = self.get_opt('url') + if ':' not in self.url: + self.url = 'http://' + self.url + self.expected_content = self.get_opt('content') + self.expected_regex = self.get_opt('regex') + if self.expected_regex: + validate_regex(self.expected_regex) + self.expected_regex = re.compile(self.expected_regex) + validate_host(self.host) + validate_port(self.port) + validate_url(self.url) + if self.get_opt('ssl'): + self.protocol = 'https' + if not self.args: + self.usage() + + def check_selenium(self, capability, url): + selenium_url = '{protocol}://{host}:{port}/{path}'\ + .format(protocol=self.protocol, \ + host=self.host, \ + port=self.port, \ + path=self.path) + log.info("Connecting to '%s' with capability '%s'", selenium_url, capability) + driver = webdriver.Remote( + command_executor=selenium_url, + desired_capabilities=getattr(DesiredCapabilities, capability) + ) + log.info("Checking url '%s'", url) + driver.get(self.url) + content = driver.page_source + if self.expected_content: + log.info("Checking url content matches '%s'", self.expected_content) + if self.expected_content not in content: + raise AssertionError('Page source content failed content match') + if self.expected_regex: + log.info("Checking url content matches regex") + if not self.expected_regex.search(content): + raise AssertionError('Page source content failed regex search') + driver.quit() + log.info("Succeeded with capability '%s' against url '%s'", capability, url) + + def run(self): + start_time = time.time() + for capability in self.args: + self.check_selenium(capability, self.url) + query_time = time.time() - start_time + log.info('Finished checks in {:.2f} secs'.format(query_time)) + + +if __name__ == '__main__': + SeleniumTest().main() From bb11b3f4ab5c534ab1bb818e1bf2dc3664e07a18 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 10:50:32 +0100 Subject: [PATCH 1724/2779] updated selenium_test.py --- selenium_test.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index 3a4118e43..e05f3d081 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -20,9 +20,15 @@ Tests a Selenium Hub / Selenoid using the given capability eg. FIREFOX, CHROME against a given URL and content (defaults to google.com) +Capabilities to check default to 'FIREFOX' and 'CHROME' +URL defaults to 'google.com' +There is no content / regex check by default + Example: - ./selenium_test.py --host [options] + ./selenium_test.py --host [options] [] + + ./selenium_test.py --host selenium-hub ./selenium_test.py --host selenium-hub FIREFOX CHROME @@ -106,7 +112,9 @@ def process_options(self): if self.get_opt('ssl'): self.protocol = 'https' if not self.args: - self.usage() + # test basic Chrome and Firefox are available + self.args.append('CHROME') + self.args.append('FIREFOX') def check_selenium(self, capability, url): selenium_url = '{protocol}://{host}:{port}/{path}'\ From fae117cd04af47c14a212aabf3ba61037628c131 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 11:03:34 +0100 Subject: [PATCH 1725/2779] updated selenium_test.py --- selenium_test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index e05f3d081..6c13c35e3 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -55,7 +55,7 @@ try: # pylint: disable=wrong-import-position from harisekhon.utils import log - from harisekhon.utils import validate_host, validate_port, validate_url, validate_regex + from harisekhon.utils import validate_host, validate_port, validate_url, validate_regex, die from harisekhon import CLI except ImportError as _: print(traceback.format_exc(), end='') @@ -130,14 +130,17 @@ def check_selenium(self, capability, url): log.info("Checking url '%s'", url) driver.get(self.url) content = driver.page_source - if self.expected_content: - log.info("Checking url content matches '%s'", self.expected_content) - if self.expected_content not in content: - raise AssertionError('Page source content failed content match') if self.expected_regex: log.info("Checking url content matches regex") if not self.expected_regex.search(content): - raise AssertionError('Page source content failed regex search') + die('ERROR: Page source content failed regex search') + elif self.expected_content: + log.info("Checking url content matches '%s'", self.expected_content) + if self.expected_content not in content: + die('ERROR: Page source content failed content match') + elif '404' in driver.title: + die('ERROR: Page title contains a 404 / error ' + + '(if this is expected, use --content / --regex instead): {}'.format(driver.title)) driver.quit() log.info("Succeeded with capability '%s' against url '%s'", capability, url) From 65ae5c0c64f3a0f0221cf8b4db92d02cf473b4fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 11:07:26 +0100 Subject: [PATCH 1726/2779] updated selenium_test.py --- selenium_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index 6c13c35e3..e57984ec7 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -130,6 +130,8 @@ def check_selenium(self, capability, url): log.info("Checking url '%s'", url) driver.get(self.url) content = driver.page_source + title = driver.title + driver.quit() if self.expected_regex: log.info("Checking url content matches regex") if not self.expected_regex.search(content): @@ -138,10 +140,9 @@ def check_selenium(self, capability, url): log.info("Checking url content matches '%s'", self.expected_content) if self.expected_content not in content: die('ERROR: Page source content failed content match') - elif '404' in driver.title: + elif '404' in title: die('ERROR: Page title contains a 404 / error ' + - '(if this is expected, use --content / --regex instead): {}'.format(driver.title)) - driver.quit() + '(if this is expected, use --content / --regex instead): {}'.format(title)) log.info("Succeeded with capability '%s' against url '%s'", capability, url) def run(self): From aea9dae435d399344bb23b36af6e30d05e7f1b5a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:16:17 +0100 Subject: [PATCH 1727/2779] updated selenium_test.py --- selenium_test.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index e57984ec7..7a2459ca8 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -17,16 +17,16 @@ """ -Tests a Selenium Hub / Selenoid using the given capability eg. FIREFOX, CHROME +Tests a Selenium Hub / Selenoid using the given browsers eg. FIREFOX, CHROME against a given URL and content (defaults to google.com) -Capabilities to check default to 'FIREFOX' and 'CHROME' +Browsers default to 'FIREFOX' and 'CHROME' if not specified URL defaults to 'google.com' There is no content / regex check by default Example: - ./selenium_test.py --host [options] [] + ./selenium_test.py --host [] [] ./selenium_test.py --host selenium-hub @@ -113,21 +113,21 @@ def process_options(self): self.protocol = 'https' if not self.args: # test basic Chrome and Firefox are available - self.args.append('CHROME') - self.args.append('FIREFOX') + self.args.append('chrome') + self.args.append('firefox') - def check_selenium(self, capability, url): + def check_selenium(self, browser): selenium_url = '{protocol}://{host}:{port}/{path}'\ .format(protocol=self.protocol, \ host=self.host, \ port=self.port, \ path=self.path) - log.info("Connecting to '%s' with capability '%s'", selenium_url, capability) + log.info("Connecting to '%s' for browser '%s'", selenium_url, browser) driver = webdriver.Remote( command_executor=selenium_url, - desired_capabilities=getattr(DesiredCapabilities, capability) + desired_capabilities=getattr(DesiredCapabilities, browser) ) - log.info("Checking url '%s'", url) + log.info("Checking url '%s'", self.url) driver.get(self.url) content = driver.page_source title = driver.title @@ -143,12 +143,12 @@ def check_selenium(self, capability, url): elif '404' in title: die('ERROR: Page title contains a 404 / error ' + '(if this is expected, use --content / --regex instead): {}'.format(title)) - log.info("Succeeded with capability '%s' against url '%s'", capability, url) + log.info("Succeeded with capability '%s' against url '%s'", browser, self.url) def run(self): start_time = time.time() - for capability in self.args: - self.check_selenium(capability, self.url) + for browser in self.args: + self.check_selenium(browser.upper()) query_time = time.time() - start_time log.info('Finished checks in {:.2f} secs'.format(query_time)) From b4485faf00545f1fa504269bce876330ef1d9906 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:17:02 +0100 Subject: [PATCH 1728/2779] updated selenium_test.py --- selenium_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/selenium_test.py b/selenium_test.py index 7a2459ca8..cdb5858ee 100755 --- a/selenium_test.py +++ b/selenium_test.py @@ -65,11 +65,11 @@ __version__ = '0.1' -class SeleniumTest(CLI): +class SeleniumHubBrowserTest(CLI): def __init__(self): # Python 2.x - super(SeleniumTest, self).__init__() + super(SeleniumHubBrowserTest, self).__init__() # Python 3.x # super().__init__() self.host = None @@ -86,7 +86,7 @@ def __init__(self): self.verbose_default = 2 def add_options(self): - super(SeleniumTest, self).add_options() + super(SeleniumHubBrowserTest, self).add_options() self.add_hostoption(name='Selenium Hub', default_port=4444) self.add_opt('-u', '--url', default=self.url_default, help='URL to use for the test (default: {})'.format(self.url_default)) @@ -95,7 +95,7 @@ def add_options(self): self.add_opt('-S', '--ssl', action='store_true', help='Use SSL to connect to Selenium Hub') def process_options(self): - super(SeleniumTest, self).process_options() + super(SeleniumHubBrowserTest, self).process_options() self.host = self.get_opt('host') self.port = self.get_opt('port') self.url = self.get_opt('url') @@ -154,4 +154,4 @@ def run(self): if __name__ == '__main__': - SeleniumTest().main() + SeleniumHubBrowserTest().main() From 223273ca6e6af6e188eaf1d0432d57f9b86ff83c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:17:12 +0100 Subject: [PATCH 1729/2779] renamed selenium_test.py to seleniumhub_browser_test.py --- selenium_test.py => seleniumhub_browser_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename selenium_test.py => seleniumhub_browser_test.py (100%) diff --git a/selenium_test.py b/seleniumhub_browser_test.py similarity index 100% rename from selenium_test.py rename to seleniumhub_browser_test.py From 43c7be738142c85d8e2a449eaa0c0e6bbad2b8d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:19:46 +0100 Subject: [PATCH 1730/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 18daac3e3..c0493de4a 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,7 @@ Environment variables are supported for convenience and also to hide credentials - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI +- ```seleniumhub_browser_test.py``` - checks [SeleniumHub](https://www.selenium.dev/documentation) by calling chrome and firefox browsers to check a given URL and content/regex match the result - Data Validation (useful in CI): - ```validate_*.py``` - validate files, directory trees and/or standard input streams - supports the following file formats: From c96abc1d55da10cf4d44943fe9a851a8ca20a512 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:21:53 +0100 Subject: [PATCH 1731/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c0493de4a..3b9f145e1 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ Environment variables are supported for convenience and also to hide credentials - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- ```seleniumhub_browser_test.py``` - checks [SeleniumHub](https://www.selenium.dev/documentation) by calling chrome and firefox browsers to check a given URL and content/regex match the result +- ```seleniumhub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - ```validate_*.py``` - validate files, directory trees and/or standard input streams - supports the following file formats: From 6cc7bdcc23af66c1cb37b4e715df1d35b7c0ba48 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:23:16 +0100 Subject: [PATCH 1732/2779] updated requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f2ceaaa9f..40a300101 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,6 +45,7 @@ python-ldap==3.2.0 python-snappy==0.5 sasl==0.2.1 sh==1.12.14 +selenium==3.141.0 # pulls in python-KrbV as a dependency which doesn't build on Mac any more # relies on python-krbV is unmaintained and unported to Python 3 # - moved to Makefile as best effort From 36b03cb3ca0d99032f9a959036a58a0fa1e07b85 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 12 May 2021 12:32:11 +0100 Subject: [PATCH 1733/2779] updated seleniumhub_browser_test.py --- seleniumhub_browser_test.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index cdb5858ee..585e0be4b 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -26,14 +26,32 @@ Example: - ./selenium_test.py --host [] [] + ./seleniumhub_browser_test.py --host [] [] - ./selenium_test.py --host selenium-hub +Where browsers are one or more of these and must be supported by the remote Selenium Hub: - ./selenium_test.py --host selenium-hub FIREFOX CHROME +ANDROID +CHROME +EDGE +FIREFOX +HTMLUNIT +HTMLUNITWITHJS +INTERNETEXPLORER +IPAD +IPHONE +OPERA +PHANTOMJS +SAFARI +WEBKITGTK - ./selenium_test.py --host selenium-hub FIREFOX CHROME --url google.com --content google - ./selenium_test.py --host selenium-hub FIREFOX CHROME --url google.com --regex 'goog.*' +Examples: + + ./seleniumhub_browser_test.py --host x.x.x.x + + ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME + + ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google + ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' """ From 4085be0e61824ad61c6318bba575b18dea3a622a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 May 2021 17:11:24 +0100 Subject: [PATCH 1734/2779] updated seleniumhub_browser_test.py --- seleniumhub_browser_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index 585e0be4b..905c0b50d 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -80,7 +80,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1' +__version__ = '0.2' class SeleniumHubBrowserTest(CLI): @@ -127,7 +127,7 @@ def process_options(self): validate_host(self.host) validate_port(self.port) validate_url(self.url) - if self.get_opt('ssl'): + if self.get_opt('ssl') or int(self.port) == 443: self.protocol = 'https' if not self.args: # test basic Chrome and Firefox are available From b5548bb26f415df8bb6c7745c03a1342327d4f70 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 May 2021 17:13:33 +0100 Subject: [PATCH 1735/2779] updated seleniumhub_browser_test.py --- seleniumhub_browser_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index 905c0b50d..3a3230925 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -99,6 +99,7 @@ def __init__(self): self.url_default = 'http://google.com' self.url = self.url_default self.expected_content = None + self.expected_content_default = 'google' self.expected_regex = None self.timeout_default = 600 self.verbose_default = 2 @@ -133,6 +134,8 @@ def process_options(self): # test basic Chrome and Firefox are available self.args.append('chrome') self.args.append('firefox') + if self.url == self.url_default: + self.expected_content = self.expected_content_default def check_selenium(self, browser): selenium_url = '{protocol}://{host}:{port}/{path}'\ From ee941e925db2c190a1109a0320f8aaf80ee0316e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 May 2021 17:14:18 +0100 Subject: [PATCH 1736/2779] updated seleniumhub_browser_test.py --- seleniumhub_browser_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index 3a3230925..1ae8920ed 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -21,8 +21,8 @@ against a given URL and content (defaults to google.com) Browsers default to 'FIREFOX' and 'CHROME' if not specified -URL defaults to 'google.com' -There is no content / regex check by default +URL defaults to 'google.com' checking for content 'google' +If you define a different URL then you must specify a --content or --regex validation otherwise none is used Example: From 3f257d161ba330cdd984cc8d51e187c71fcdb212 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:48:17 +0100 Subject: [PATCH 1737/2779] added --hub-url option --- seleniumhub_browser_test.py | 46 ++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/seleniumhub_browser_test.py b/seleniumhub_browser_test.py index 1ae8920ed..84456583d 100755 --- a/seleniumhub_browser_test.py +++ b/seleniumhub_browser_test.py @@ -26,7 +26,9 @@ Example: - ./seleniumhub_browser_test.py --host [] [] + ./selenium_hub_browser_test.py --host [] [] + + ./selenium_hub_browser_test.py --hub-url https://:4444/wd/hub/ [] [] Where browsers are one or more of these and must be supported by the remote Selenium Hub: @@ -46,12 +48,12 @@ Examples: - ./seleniumhub_browser_test.py --host x.x.x.x + ./selenium_hub_browser_test.py --host x.x.x.x - ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME + ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME - ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google - ./seleniumhub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' + ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google + ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' """ @@ -80,7 +82,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2' +__version__ = '0.3' class SeleniumHubBrowserTest(CLI): @@ -96,6 +98,7 @@ def __init__(self): self.name = 'Selenium Hub' self.default_port = 80 self.path = 'wd/hub' + self.hub_url = None self.url_default = 'http://google.com' self.url = self.url_default self.expected_content = None @@ -107,6 +110,7 @@ def __init__(self): def add_options(self): super(SeleniumHubBrowserTest, self).add_options() self.add_hostoption(name='Selenium Hub', default_port=4444) + self.add_opt('-U', '--hub-url', help='Selenium Hub URL') self.add_opt('-u', '--url', default=self.url_default, help='URL to use for the test (default: {})'.format(self.url_default)) self.add_opt('-c', '--content', help='URL content to expect') @@ -115,8 +119,21 @@ def add_options(self): def process_options(self): super(SeleniumHubBrowserTest, self).process_options() - self.host = self.get_opt('host') - self.port = self.get_opt('port') + self.hub_url = self.get_opt('hub_url') + if self.hub_url: + validate_url(self.hub_url, 'hub') + else: + self.host = self.get_opt('host') + self.port = self.get_opt('port') + validate_host(self.host) + validate_port(self.port) + if self.get_opt('ssl') or int(self.port) == 443: + self.protocol = 'https' + self.hub_url = '{protocol}://{host}:{port}/{path}'\ + .format(protocol=self.protocol, \ + host=self.host, \ + port=self.port, \ + path=self.path) self.url = self.get_opt('url') if ':' not in self.url: self.url = 'http://' + self.url @@ -125,11 +142,7 @@ def process_options(self): if self.expected_regex: validate_regex(self.expected_regex) self.expected_regex = re.compile(self.expected_regex) - validate_host(self.host) - validate_port(self.port) validate_url(self.url) - if self.get_opt('ssl') or int(self.port) == 443: - self.protocol = 'https' if not self.args: # test basic Chrome and Firefox are available self.args.append('chrome') @@ -138,14 +151,9 @@ def process_options(self): self.expected_content = self.expected_content_default def check_selenium(self, browser): - selenium_url = '{protocol}://{host}:{port}/{path}'\ - .format(protocol=self.protocol, \ - host=self.host, \ - port=self.port, \ - path=self.path) - log.info("Connecting to '%s' for browser '%s'", selenium_url, browser) + log.info("Connecting to '%s' for browser '%s'", self.hub_url, browser) driver = webdriver.Remote( - command_executor=selenium_url, + command_executor=self.hub_url, desired_capabilities=getattr(DesiredCapabilities, browser) ) log.info("Checking url '%s'", self.url) From dc11c44e5b8935dd3b95f5fa82181b2ff265c5b0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:48:24 +0100 Subject: [PATCH 1738/2779] renamed seleniumhub_browser_test.py to selenium_hub_browser_test.py --- seleniumhub_browser_test.py => selenium_hub_browser_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename seleniumhub_browser_test.py => selenium_hub_browser_test.py (100%) diff --git a/seleniumhub_browser_test.py b/selenium_hub_browser_test.py similarity index 100% rename from seleniumhub_browser_test.py rename to selenium_hub_browser_test.py From e87d99685efa3a26523c4391b17de2e0ad7b7c79 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:50:54 +0100 Subject: [PATCH 1739/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 40a300101..a64f54236 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ GitPython==2.1.15 happybase==1.0.0 humanize==0.5.1 impyla==0.16.0 -Jinja2==2.10.1 +jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 #MarkupSafe==0.23 From b1fa126f6c0044c05bd30aa1433409e18306d5e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:52:05 +0100 Subject: [PATCH 1740/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b9f145e1..6cb10d2f7 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ Environment variables are supported for convenience and also to hide credentials - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- ```seleniumhub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result +- ```selenium_hub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - ```validate_*.py``` - validate files, directory trees and/or standard input streams - supports the following file formats: From 583296e3dc706f94ccd6273f1817106e9d4a4078 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 08:57:43 +0100 Subject: [PATCH 1741/2779] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 84456583d..7ea5f1860 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -96,7 +96,6 @@ def __init__(self): self.port = None self.protocol = 'http' self.name = 'Selenium Hub' - self.default_port = 80 self.path = 'wd/hub' self.hub_url = None self.url_default = 'http://google.com' @@ -110,7 +109,7 @@ def __init__(self): def add_options(self): super(SeleniumHubBrowserTest, self).add_options() self.add_hostoption(name='Selenium Hub', default_port=4444) - self.add_opt('-U', '--hub-url', help='Selenium Hub URL') + self.add_opt('-U', '--hub-url', help='Selenium Hub URL (takes priority over --host/--port/--ssl)') self.add_opt('-u', '--url', default=self.url_default, help='URL to use for the test (default: {})'.format(self.url_default)) self.add_opt('-c', '--content', help='URL content to expect') From aacd839656f8c7c6435cbadc885b86df5caf6f1b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:02:18 +0100 Subject: [PATCH 1742/2779] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 7ea5f1860..462973872 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -136,18 +136,18 @@ def process_options(self): self.url = self.get_opt('url') if ':' not in self.url: self.url = 'http://' + self.url + validate_url(self.url) self.expected_content = self.get_opt('content') self.expected_regex = self.get_opt('regex') if self.expected_regex: validate_regex(self.expected_regex) self.expected_regex = re.compile(self.expected_regex) - validate_url(self.url) + elif self.url == self.url_default: + self.expected_content = self.expected_content_default if not self.args: # test basic Chrome and Firefox are available self.args.append('chrome') self.args.append('firefox') - if self.url == self.url_default: - self.expected_content = self.expected_content_default def check_selenium(self, browser): log.info("Connecting to '%s' for browser '%s'", self.hub_url, browser) From 90dbc0823dbfe9f3c96f5cc4375ff4faa9d5d5cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:05:42 +0100 Subject: [PATCH 1743/2779] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 462973872..7a5472374 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -74,9 +74,8 @@ sys.path.append(libdir) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log - from harisekhon.utils import validate_host, validate_port, validate_url, validate_regex, die from harisekhon import CLI + from harisekhon.utils import log, validate_host, validate_port, validate_url, validate_regex, die except ImportError as _: print(traceback.format_exc(), end='') sys.exit(4) From b09f912723fcf1144955c032831924aca3ed251d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:12:01 +0100 Subject: [PATCH 1744/2779] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 7a5472374..00734c969 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -170,7 +170,7 @@ def check_selenium(self, browser): elif '404' in title: die('ERROR: Page title contains a 404 / error ' + '(if this is expected, use --content / --regex instead): {}'.format(title)) - log.info("Succeeded with capability '%s' against url '%s'", browser, self.url) + log.info("Succeeded for browser '%s' against url '%s'", browser, self.url) def run(self): start_time = time.time() From 8ecff7acdd9bb0f335ca31deddbbf6f6bb9a303c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:23:45 +0100 Subject: [PATCH 1745/2779] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 00734c969..8d498cc9a 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -169,7 +169,7 @@ def check_selenium(self, browser): die('ERROR: Page source content failed content match') elif '404' in title: die('ERROR: Page title contains a 404 / error ' + - '(if this is expected, use --content / --regex instead): {}'.format(title)) + '(if this is expected, specify --content / --regex to check instead): {}'.format(title)) log.info("Succeeded for browser '%s' against url '%s'", browser, self.url) def run(self): From 491e8723da7b61d45f7366ae958ab23673bee1fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 09:31:14 +0100 Subject: [PATCH 1746/2779] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index 8d498cc9a..f67c45327 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -28,7 +28,7 @@ ./selenium_hub_browser_test.py --host [] [] - ./selenium_hub_browser_test.py --hub-url https://:4444/wd/hub/ [] [] + ./selenium_hub_browser_test.py --hub-url http://:4444/wd/hub/ [] [] Where browsers are one or more of these and must be supported by the remote Selenium Hub: @@ -55,6 +55,8 @@ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' + +Tested on Selenium Grid Hub v.3.141.59, and Selenoid 1.10.1 """ from __future__ import absolute_import From 37699e74f1b787268cdd3ba076c3adbe090b14c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 10:55:34 +0100 Subject: [PATCH 1747/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 7b69c6157..528aa8999 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 7b69c61571a3a27b786884999fd0937a8150cc78 +Subproject commit 528aa89996f39b5d6c4d0df41404d9ca4c6fff3d From 717dfde4c3e1593f5f79cb1cc17f3df54cf8d78e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 10:55:34 +0100 Subject: [PATCH 1748/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 65631842d..964ff75d9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 65631842daa333da9e8f5ca4ea26f30d380e53b6 +Subproject commit 964ff75d90eb7337a998bbab4e832ca7307b4f04 From 5723131b1e4865e0eab85e59461352be653cb55b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 May 2021 10:55:35 +0100 Subject: [PATCH 1749/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a9ebed85a..4775e138b 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a9ebed85a6753ef025b8d1c49fdb6a466bb66727 +Subproject commit 4775e138bbb851f9157e318060f420389d3b695d From 9a26ae3df818bb9ecb03bac0aa58164c82214012 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 May 2021 19:14:32 +0100 Subject: [PATCH 1750/2779] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index f67c45327..f9fd17824 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -169,6 +169,10 @@ def check_selenium(self, browser): log.info("Checking url content matches '%s'", self.expected_content) if self.expected_content not in content: die('ERROR: Page source content failed content match') + # not really recommended but in this case we cannot predict + # what to expect on a random url if not specified by --content/--regex: + # + # https://www.selenium.dev/documentation/en/worst_practices/http_response_codes/ elif '404' in title: die('ERROR: Page title contains a 404 / error ' + '(if this is expected, specify --content / --regex to check instead): {}'.format(title)) From 198c10ae442105e2ae16c58dbcd7c090787d4abe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 May 2021 19:22:32 +0100 Subject: [PATCH 1751/2779] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index f9fd17824..cb8bebd99 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -170,7 +170,7 @@ def check_selenium(self, browser): if self.expected_content not in content: die('ERROR: Page source content failed content match') # not really recommended but in this case we cannot predict - # what to expect on a random url if not specified by --content/--regex: + # what to expect on a random url if not specified by --content/--regex (provided in the default test case) # # https://www.selenium.dev/documentation/en/worst_practices/http_response_codes/ elif '404' in title: From 5d3dc09060b7336ad2536cf6ab63d42380e502ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 May 2021 19:23:12 +0100 Subject: [PATCH 1752/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aedc339bf..dfef9fec9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,4 +57,4 @@ thriftpy==0.3.9 toml==0.10.0 xmltodict==0.10.2 yamllint==1.15.0 -pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability +#pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability. update: this breaks Python 3.5 build where this requirement is not found From cdc3e04c66329659930df52b2f78712afd06cc87 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 May 2021 11:06:34 +0100 Subject: [PATCH 1753/2779] updated selenium_hub_browser_test.py --- selenium_hub_browser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index cb8bebd99..8b0242b83 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -56,7 +56,7 @@ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' -Tested on Selenium Grid Hub v.3.141.59, and Selenoid 1.10.1 +Tested on Selenium Grid Hub v.3.141.59, v4.0.0 and Selenoid 1.10.1 """ from __future__ import absolute_import From 2683ba9b7564b178a589e76cd3f7bc1cae940ed9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:10:10 +0100 Subject: [PATCH 1754/2779] added main.py --- gcp_cloud_function_ifconfig/main.py | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 gcp_cloud_function_ifconfig/main.py diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py new file mode 100755 index 000000000..ba3089e20 --- /dev/null +++ b/gcp_cloud_function_ifconfig/main.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access +routing via specified VPC Network to using default NAT Gateway + +Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules + +The HTTP request is irrelevant, although we could pass the website to query via an arg in which case this would just act as a proxy + +Tested on GCP Cloud Functions with Python 3.9 + +""" + +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python + +import requests + +def main(request): + """Responds to any HTTP request. + Args: + request (flask.Request): HTTP request object. + Returns: + The response text or any set of values that can be turned into a + Response object using + `make_response `. + """ + r = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing + status_code = r.status_code + status_message = r.reason + content = r.text + return "{} {}\n\n{}".format(status_code, status_message, content) From 5bacef94a5296245b440c98975fa8d9cb7bdfed2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:15:00 +0100 Subject: [PATCH 1755/2779] updated main.py --- gcp_cloud_function_ifconfig/main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py index ba3089e20..a2f4e6783 100755 --- a/gcp_cloud_function_ifconfig/main.py +++ b/gcp_cloud_function_ifconfig/main.py @@ -22,7 +22,8 @@ Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules -The HTTP request is irrelevant, although we could pass the website to query via an arg in which case this would just act as a proxy +The HTTP request is irrelevant, although we could pass the website to query via an arg +in which case this would just act as a proxy. See the adjacent gcp_cloud_function_proxy/main.py Tested on GCP Cloud Functions with Python 3.9 @@ -32,7 +33,7 @@ import requests -def main(request): +def main(_): """Responds to any HTTP request. Args: request (flask.Request): HTTP request object. @@ -41,8 +42,8 @@ def main(request): Response object using `make_response `. """ - r = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing - status_code = r.status_code - status_message = r.reason - content = r.text + req = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing + status_code = req.status_code + status_message = req.reason + content = req.text return "{} {}\n\n{}".format(status_code, status_message, content) From f26a380d667ad5bd4490ef422521e9dfadfc8007 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:26:56 +0100 Subject: [PATCH 1756/2779] added main.py --- gcp_cloud_function_proxy/main.py | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 gcp_cloud_function_proxy/main.py diff --git a/gcp_cloud_function_proxy/main.py b/gcp_cloud_function_proxy/main.py new file mode 100755 index 000000000..3c6209ca6 --- /dev/null +++ b/gcp_cloud_function_proxy/main.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access +routing via specified VPC Network to using default NAT Gateway + +Example usage: + +Check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules + +Test request examples: + + { "url": "http://ifconfig.co/json" } + +defaults to http:// if not specified: + + { "url": "ifconfig.co/json" } + + +Tested on GCP Cloud Functions with Python 3.9 + +""" + +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python + +import json +import requests + +def main(request): + """Responds to any HTTP request. + Args: + request (flask.Request): HTTP request object. + Returns: + The response text or any set of values that can be turned into a + Response object using + `make_response `. + """ + data = json.loads(request.data) + url = data['url'] + if '://' not in url: + url = 'http://' + url + req = requests.get(url) + status_code = req.status_code + status_message = req.reason + content = req.text + return "{} {}\n\n{}".format(status_code, status_message, content) From e8f457b5eac3d165f0b3d15aa2881e543337a155 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:30:29 +0100 Subject: [PATCH 1757/2779] updated main.py --- gcp_cloud_function_ifconfig/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py index a2f4e6783..309bc58c0 100755 --- a/gcp_cloud_function_ifconfig/main.py +++ b/gcp_cloud_function_ifconfig/main.py @@ -22,13 +22,18 @@ Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules -The HTTP request is irrelevant, although we could pass the website to query via an arg -in which case this would just act as a proxy. See the adjacent gcp_cloud_function_proxy/main.py +The HTTP request is irrelevant, just pass an empty JSON document '{}', although we could pass the website to query +in a field in which case this would just act as a proxy. + +See Also: gcp_cloud_function_proxy/main.py + Tested on GCP Cloud Functions with Python 3.9 """ +# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python + # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python import requests From d739b4e24301ec0c278b638cda34e0158b70d66a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:31:14 +0100 Subject: [PATCH 1758/2779] updated main.py --- gcp_cloud_function_proxy/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gcp_cloud_function_proxy/main.py b/gcp_cloud_function_proxy/main.py index 3c6209ca6..9e525e72b 100755 --- a/gcp_cloud_function_proxy/main.py +++ b/gcp_cloud_function_proxy/main.py @@ -37,6 +37,8 @@ """ +# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python + # https://cloud.google.com/functions/docs/writing/specifying-dependencies-python import json From 12e626c93b05ea461ecd3e0325a670661304154a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:32:27 +0100 Subject: [PATCH 1759/2779] added requirements.txt --- gcp_cloud_function_proxy/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 gcp_cloud_function_proxy/requirements.txt diff --git a/gcp_cloud_function_proxy/requirements.txt b/gcp_cloud_function_proxy/requirements.txt new file mode 100644 index 000000000..775d29ac6 --- /dev/null +++ b/gcp_cloud_function_proxy/requirements.txt @@ -0,0 +1,2 @@ +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python +requests==2.24.0 From 914f86d3dfabaf5b33b1b544a5db405a89896b2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:32:42 +0100 Subject: [PATCH 1760/2779] added requirements.txt --- gcp_cloud_function_ifconfig/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 gcp_cloud_function_ifconfig/requirements.txt diff --git a/gcp_cloud_function_ifconfig/requirements.txt b/gcp_cloud_function_ifconfig/requirements.txt new file mode 100644 index 000000000..775d29ac6 --- /dev/null +++ b/gcp_cloud_function_ifconfig/requirements.txt @@ -0,0 +1,2 @@ +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python +requests==2.24.0 From e68719a6d74ff5f868c57f800b8cd0c8bec51bc2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:33:24 +0100 Subject: [PATCH 1761/2779] added .gcloudignore --- gcp_cloud_function_proxy/.gcloudignore | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_proxy/.gcloudignore diff --git a/gcp_cloud_function_proxy/.gcloudignore b/gcp_cloud_function_proxy/.gcloudignore new file mode 100644 index 000000000..8f605abc2 --- /dev/null +++ b/gcp_cloud_function_proxy/.gcloudignore @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore + +.git +.gcloudignore +deploy.sh +test/ +tests/ From 9ba9d0ba5e610ee1fc019b2fa08469dfb299814b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:33:35 +0100 Subject: [PATCH 1762/2779] added .gcloudignore --- gcp_cloud_function_ifconfig/.gcloudignore | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_ifconfig/.gcloudignore diff --git a/gcp_cloud_function_ifconfig/.gcloudignore b/gcp_cloud_function_ifconfig/.gcloudignore new file mode 100644 index 000000000..8f605abc2 --- /dev/null +++ b/gcp_cloud_function_ifconfig/.gcloudignore @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore + +.git +.gcloudignore +deploy.sh +test/ +tests/ From de6f67df8583ecfc2c3354acd461897a10689323 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:43:59 +0100 Subject: [PATCH 1763/2779] updated deploy.sh --- gcp_cloud_function_sql_export/deploy.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh index 1465dbc56..62857a9c7 100755 --- a/gcp_cloud_function_sql_export/deploy.sh +++ b/gcp_cloud_function_sql_export/deploy.sh @@ -19,9 +19,11 @@ srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$srcdir" -project="$(gcloud config list --format="value(core.project)")" -region="$(gcloud config list --format="value(compute.region)")" -region="${region:-${GOOGLE_REGION:-europe-west1}}" # not available in all regions yet +# needed to define the $service_account further down +project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config list --format="value(core.project)")}" + +region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet name="cloud-sql-backups" topic="cloud-sql-backups" From 43eaa6fc890a70bded7e86fcf9ecddb618a9aa07 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:45:26 +0100 Subject: [PATCH 1764/2779] added deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 gcp_cloud_function_ifconfig/deploy.sh diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh new file mode 100755 index 000000000..c103f9efe --- /dev/null +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cd "$srcdir" + +name="ifconfig" + +# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com +# for serverless VPC access to resources using their Private IPs +# since we're only accessing the SQL Admin API we don't need this +#vpc_connector="ifconfig" + +gcloud functions deploy "$name" --trigger-http \ + --security-level=secure-always \ + --runtime python39 \ + --entry-point main \ + --memory 128MB \ + --timeout 60 \ + --quiet # don't prompt to --allow-unauthenticated + #--max-instances 1 + #--vpc-connector "$vpc_connector" From 004d3b13ad4c6c6f03ac33f7e31b948627fcbfd9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 16:59:51 +0100 Subject: [PATCH 1765/2779] updated deploy.sh --- gcp_cloud_function_sql_export/deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh index 62857a9c7..64705ad5c 100755 --- a/gcp_cloud_function_sql_export/deploy.sh +++ b/gcp_cloud_function_sql_export/deploy.sh @@ -22,6 +22,7 @@ cd "$srcdir" # needed to define the $service_account further down project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config list --format="value(core.project)")}" +# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet From 6c25b875d833c6922a1ad220dced49a41ba93721 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 17:02:59 +0100 Subject: [PATCH 1766/2779] updated deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh index c103f9efe..aa1fbe03b 100755 --- a/gcp_cloud_function_ifconfig/deploy.sh +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -21,17 +21,22 @@ cd "$srcdir" name="ifconfig" +# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment +region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet + # https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com # for serverless VPC access to resources using their Private IPs # since we're only accessing the SQL Admin API we don't need this -#vpc_connector="ifconfig" +#vpc_connector="my-vpc-connector" gcloud functions deploy "$name" --trigger-http \ --security-level=secure-always \ --runtime python39 \ --entry-point main \ --memory 128MB \ + --region "$region" \ --timeout 60 \ --quiet # don't prompt to --allow-unauthenticated - #--max-instances 1 - #--vpc-connector "$vpc_connector" + #--vpc-connector "$vpc_connector" \ + #--max-instances 1 \ From db8f71b08872c8dec774e6fd9284a35a7446f9b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 17:08:50 +0100 Subject: [PATCH 1767/2779] added deploy.sh --- gcp_cloud_function_proxy/deploy.sh | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 gcp_cloud_function_proxy/deploy.sh diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh new file mode 100755 index 000000000..d77f41c04 --- /dev/null +++ b/gcp_cloud_function_proxy/deploy.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/pytools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cd "$srcdir" + +name="proxy" + +# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment +region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet + +# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com +# for serverless VPC access to resources using their Private IPs +# since we're only accessing the SQL Admin API we don't need this +#vpc_connector="my-vpc-connector" + +gcloud functions deploy "$name" --trigger-http \ + --security-level=secure-always \ + --runtime python39 \ + --entry-point main \ + --memory 128MB \ + --region "$region" \ + --timeout 60 \ + --quiet # don't prompt to --allow-unauthenticated + # routes private traffic only by default - need to change this if you want to + # use the Cloud NAT gateway from the VPC for Firewall rules purposes + #--vpc-connector "$vpc_connector" \ + #--max-instances 1 From 8ace6d77ec56a353f8284e4ff3babca347cdd04f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 17:11:03 +0100 Subject: [PATCH 1768/2779] added README.md --- gcp_cloud_function_ifconfig/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 gcp_cloud_function_ifconfig/README.md diff --git a/gcp_cloud_function_ifconfig/README.md b/gcp_cloud_function_ifconfig/README.md new file mode 100644 index 000000000..d75717529 --- /dev/null +++ b/gcp_cloud_function_ifconfig/README.md @@ -0,0 +1,14 @@ +Google Cloud Function - SQL Backup Exporter to GCS +===================== + +Queries http://ifconfig.co from GCF to check the routing and external IP being used eg. for comparison with Cloudflare / Firewall rules + +- `main.py` - the code +- `requirements.txt` - the pip modules to bootstrap +- `deploy.sh` - upload the code and deps + +Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: + +``` +./deploy.sh +``` From 6e27129b2fb7ab6dbd74e5c38f845b9184656819 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 24 May 2021 17:12:22 +0100 Subject: [PATCH 1769/2779] added README.md --- gcp_cloud_function_proxy/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 gcp_cloud_function_proxy/README.md diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md new file mode 100644 index 000000000..c0368b1c2 --- /dev/null +++ b/gcp_cloud_function_proxy/README.md @@ -0,0 +1,19 @@ +Google Cloud Function - SQL Backup Exporter to GCS +===================== + +Queries a given URL from GCF to check connectivity eg. for testing with Cloudflare / Firewall rules + +The query string should be like so: +``` +{"url": "http://ifconfig.co/json"} +``` + +- `main.py` - the code +- `requirements.txt` - the pip modules to bootstrap +- `deploy.sh` - upload the code and deps + +Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: + +``` +./deploy.sh +``` From 9bf8e665e455df4c5021f43abc4a067d8f4bb235 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:32:54 +0100 Subject: [PATCH 1770/2779] updated README.md --- gcp_cloud_function_proxy/README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md index c0368b1c2..78c6ff1f1 100644 --- a/gcp_cloud_function_proxy/README.md +++ b/gcp_cloud_function_proxy/README.md @@ -1,13 +1,22 @@ -Google Cloud Function - SQL Backup Exporter to GCS +Google Cloud Function - Proxy ===================== Queries a given URL from GCF to check connectivity eg. for testing with Cloudflare / Firewall rules -The query string should be like so: +Query content: ``` {"url": "http://ifconfig.co/json"} ``` +Response: + +``` +200 OK + + +``` + + - `main.py` - the code - `requirements.txt` - the pip modules to bootstrap - `deploy.sh` - upload the code and deps From b6118c024e33768845dc031a7bb0e4a23e442fac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:33:36 +0100 Subject: [PATCH 1771/2779] updated README.md --- gcp_cloud_function_proxy/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md index 78c6ff1f1..9cee20eb8 100644 --- a/gcp_cloud_function_proxy/README.md +++ b/gcp_cloud_function_proxy/README.md @@ -11,11 +11,20 @@ Query content: Response: ``` -200 OK + ``` +eg. + +``` +200 OK + + +... +``` + - `main.py` - the code - `requirements.txt` - the pip modules to bootstrap From 29829f1c0e512f9206b2b7dd5da71860e6e8ba3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:34:49 +0100 Subject: [PATCH 1772/2779] updated README.md --- gcp_cloud_function_proxy/README.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md index 9cee20eb8..1a4c2465a 100644 --- a/gcp_cloud_function_proxy/README.md +++ b/gcp_cloud_function_proxy/README.md @@ -8,24 +8,14 @@ Query content: {"url": "http://ifconfig.co/json"} ``` -Response: - -``` - - - -``` - -eg. +Response is HTTP status code and message, blank line and then the content: ``` 200 OK - -... + ``` - - `main.py` - the code - `requirements.txt` - the pip modules to bootstrap - `deploy.sh` - upload the code and deps From 59750ea035f53f7af608ae13d16759ae654c5d9f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:37:53 +0100 Subject: [PATCH 1773/2779] added Makefile --- gcp_cloud_function_sql_export/Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_sql_export/Makefile diff --git a/gcp_cloud_function_sql_export/Makefile b/gcp_cloud_function_sql_export/Makefile new file mode 100644 index 000000000..656b78879 --- /dev/null +++ b/gcp_cloud_function_sql_export/Makefile @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Kubernetes-templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +SHELL = /usr/bin/env bash + +.PHONY: default +default: deploy + @: + +.PHONY: deploy +deploy: + @./deploy.sh From d196c0e4b48b617b88e07f3c8e7f8b77dc170f94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:38:08 +0100 Subject: [PATCH 1774/2779] added Makefile --- gcp_cloud_function_ifconfig/Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_ifconfig/Makefile diff --git a/gcp_cloud_function_ifconfig/Makefile b/gcp_cloud_function_ifconfig/Makefile new file mode 100644 index 000000000..656b78879 --- /dev/null +++ b/gcp_cloud_function_ifconfig/Makefile @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Kubernetes-templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +SHELL = /usr/bin/env bash + +.PHONY: default +default: deploy + @: + +.PHONY: deploy +deploy: + @./deploy.sh From 88a406f911885efa2fedeba388e45b215a036e10 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 11:38:17 +0100 Subject: [PATCH 1775/2779] added Makefile --- gcp_cloud_function_proxy/Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 gcp_cloud_function_proxy/Makefile diff --git a/gcp_cloud_function_proxy/Makefile b/gcp_cloud_function_proxy/Makefile new file mode 100644 index 000000000..656b78879 --- /dev/null +++ b/gcp_cloud_function_proxy/Makefile @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Kubernetes-templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +SHELL = /usr/bin/env bash + +.PHONY: default +default: deploy + @: + +.PHONY: deploy +deploy: + @./deploy.sh From 68d0b7ad41d083b5e476008c29a3ffea0ea59109 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:13:45 +0100 Subject: [PATCH 1776/2779] updated README.md --- gcp_cloud_function_ifconfig/README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_ifconfig/README.md b/gcp_cloud_function_ifconfig/README.md index d75717529..be6e431a1 100644 --- a/gcp_cloud_function_ifconfig/README.md +++ b/gcp_cloud_function_ifconfig/README.md @@ -1,4 +1,4 @@ -Google Cloud Function - SQL Backup Exporter to GCS +Google Cloud Function - ifconfig ===================== Queries http://ifconfig.co from GCF to check the routing and external IP being used eg. for comparison with Cloudflare / Firewall rules @@ -7,6 +7,31 @@ Queries http://ifconfig.co from GCF to check the routing and external IP being u - `requirements.txt` - the pip modules to bootstrap - `deploy.sh` - upload the code and deps +Response is HTTP status code and message, then the raw JSON results + +``` +200 OK + +{ + "ip": "...", + "ip_decimal": ... , + "country": "United States", + "country_iso": "US", + "country_eu": false, + "latitude": 37.751, + "longitude": -97.822, + "time_zone": "America/Chicago", + "asn": "AS15169", + "asn_org": "GOOGLE", + "hostname": "ipv6.gae.googleusercontent.com", + "user_agent": { + "product": "python-requests", + "version": "2.24.0", + "raw_value": "python-requests/2.24.0" + } +} +``` + Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: ``` From abead35e94e5e3a54ef456dd9a4fc5974e9e2214 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:15:56 +0100 Subject: [PATCH 1777/2779] updated README.md --- gcp_cloud_function_ifconfig/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/gcp_cloud_function_ifconfig/README.md b/gcp_cloud_function_ifconfig/README.md index be6e431a1..0485fcf08 100644 --- a/gcp_cloud_function_ifconfig/README.md +++ b/gcp_cloud_function_ifconfig/README.md @@ -13,8 +13,8 @@ Response is HTTP status code and message, then the raw JSON results 200 OK { - "ip": "...", - "ip_decimal": ... , + "ip": "1.2.3.4", + "ip_decimal": 1234567890, "country": "United States", "country_iso": "US", "country_eu": false, @@ -32,6 +32,12 @@ Response is HTTP status code and message, then the raw JSON results } ``` +For IPv6 the format will be more like: +``` + "ip": "1234:5678:9012:34::a", + "ip_decimal": 12345678901234567890123456789012345678, +``` + Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: ``` From fccb4e40c3c70d7cb2ceb2970840551a5221d4a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:20:27 +0100 Subject: [PATCH 1778/2779] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6cb10d2f7..2a48b68fb 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,8 @@ Environment variables are supported for convenience and also to hide credentials - [Google Cloud Platform](https://cloud.google.com/): - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - [GCF](https://cloud.google.com/functions) Python function to debug your Cloud Function public networking by determining which IP address your cloud function is seen as coming from + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - [GCF](https://cloud.google.com/functions) Python function to debug your Cloud Function networking by querying a given URL to check accessibility, returning the HTTP status code and content - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From 6f28bdba976126c2b973aece6c7e99e4fad3831a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:23:49 +0100 Subject: [PATCH 1779/2779] updated README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2a48b68fb..467cb7c86 100644 --- a/README.md +++ b/README.md @@ -188,10 +188,11 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - [GCF](https://cloud.google.com/functions) Python function to run [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - [GCF](https://cloud.google.com/functions) Python function to debug your Cloud Function public networking by determining which IP address your cloud function is seen as coming from - - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - [GCF](https://cloud.google.com/functions) Python function to debug your Cloud Function networking by querying a given URL to check accessibility, returning the HTTP status code and content + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From ef9be89b6f36f0689b16d72f45de530415c1482e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 12:23:58 +0100 Subject: [PATCH 1780/2779] updated README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 467cb7c86..d36cef7cc 100644 --- a/README.md +++ b/README.md @@ -189,10 +189,10 @@ Environment variables are supported for convenience and also to hide credentials - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` From 979acb46c157840ad63779076d13218196acf15b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 17:56:05 +0100 Subject: [PATCH 1781/2779] updated deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh index aa1fbe03b..015234fb7 100755 --- a/gcp_cloud_function_ifconfig/deploy.sh +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -21,9 +21,13 @@ cd "$srcdir" name="ifconfig" +# Cloud Functions not available in all regions yet: +# +# https://cloud.google.com/functions/docs/locations +# # gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" -region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com # for serverless VPC access to resources using their Private IPs From 432d2a79eb04a065ebc11a87d642d308c9411131 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 17:56:19 +0100 Subject: [PATCH 1782/2779] updated deploy.sh --- gcp_cloud_function_proxy/deploy.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh index d77f41c04..da82e737c 100755 --- a/gcp_cloud_function_proxy/deploy.sh +++ b/gcp_cloud_function_proxy/deploy.sh @@ -21,9 +21,13 @@ cd "$srcdir" name="proxy" +# Cloud Functions not available in all regions yet: +# +# https://cloud.google.com/functions/docs/locations +# # gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" -region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com # for serverless VPC access to resources using their Private IPs From 5afaf5ffa7403c6f19435f213dadd2dda97c52e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 May 2021 17:56:48 +0100 Subject: [PATCH 1783/2779] updated deploy.sh --- gcp_cloud_function_sql_export/deploy.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh index 64705ad5c..3dd2c44f5 100755 --- a/gcp_cloud_function_sql_export/deploy.sh +++ b/gcp_cloud_function_sql_export/deploy.sh @@ -22,9 +22,13 @@ cd "$srcdir" # needed to define the $service_account further down project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config list --format="value(core.project)")}" +# Cloud Functions not available in all regions yet: +# +# https://cloud.google.com/functions/docs/locations +# # gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" -region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # not available in all regions yet +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" name="cloud-sql-backups" topic="cloud-sql-backups" From d4bb08e84801dc3a664137eb553d6ce8de90520f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 May 2021 17:31:19 +0100 Subject: [PATCH 1784/2779] updated deploy.sh --- gcp_cloud_function_proxy/deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh index da82e737c..9854672cd 100755 --- a/gcp_cloud_function_proxy/deploy.sh +++ b/gcp_cloud_function_proxy/deploy.sh @@ -45,4 +45,5 @@ gcloud functions deploy "$name" --trigger-http \ # routes private traffic only by default - need to change this if you want to # use the Cloud NAT gateway from the VPC for Firewall rules purposes #--vpc-connector "$vpc_connector" \ + #--egress-settings all # sends all traffic through connector to exit via network's Cloud NAT IP #--max-instances 1 From 174a6f2867c5ea027a6436b45a7e9db86fdf513c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 May 2021 17:31:30 +0100 Subject: [PATCH 1785/2779] updated deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh index 015234fb7..f24a04a45 100755 --- a/gcp_cloud_function_ifconfig/deploy.sh +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -42,5 +42,8 @@ gcloud functions deploy "$name" --trigger-http \ --region "$region" \ --timeout 60 \ --quiet # don't prompt to --allow-unauthenticated + # routes private traffic only by default - need to change this if you want to + # use the Cloud NAT gateway from the VPC for Firewall rules purposes #--vpc-connector "$vpc_connector" \ + #--egress-settings all # sends all traffic through connector to exit via network's Cloud NAT IP #--max-instances 1 \ From 131bf5caabb4a223a3fd2090f6201639763bc8eb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 May 2021 18:45:20 +0100 Subject: [PATCH 1786/2779] updated deploy.sh --- gcp_cloud_function_ifconfig/deploy.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh index f24a04a45..6f7fc0ab3 100755 --- a/gcp_cloud_function_ifconfig/deploy.sh +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -34,6 +34,13 @@ region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # since we're only accessing the SQL Admin API we don't need this #vpc_connector="my-vpc-connector" +opts=() +if [ -n "${vpc_connector:-}" ]; then + # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules + opts+=(--vpc-connector "$vpc_connector" --egress-settings all) +fi + +set -x gcloud functions deploy "$name" --trigger-http \ --security-level=secure-always \ --runtime python39 \ @@ -41,9 +48,6 @@ gcloud functions deploy "$name" --trigger-http \ --memory 128MB \ --region "$region" \ --timeout 60 \ + "${opts[@]}" \ --quiet # don't prompt to --allow-unauthenticated - # routes private traffic only by default - need to change this if you want to - # use the Cloud NAT gateway from the VPC for Firewall rules purposes - #--vpc-connector "$vpc_connector" \ - #--egress-settings all # sends all traffic through connector to exit via network's Cloud NAT IP #--max-instances 1 \ From d70585017e42de536a7053b991e1e6fafdd9bce7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 27 May 2021 19:00:00 +0100 Subject: [PATCH 1787/2779] updated deploy.sh --- gcp_cloud_function_proxy/deploy.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh index 9854672cd..a583c1d2e 100755 --- a/gcp_cloud_function_proxy/deploy.sh +++ b/gcp_cloud_function_proxy/deploy.sh @@ -34,6 +34,13 @@ region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" # since we're only accessing the SQL Admin API we don't need this #vpc_connector="my-vpc-connector" +opts=() +if [ -n "${vpc_connector:-}" ]; then + # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules + opts+=(--vpc-connector "$vpc_connector" --egress-settings all) +fi + +set -x gcloud functions deploy "$name" --trigger-http \ --security-level=secure-always \ --runtime python39 \ @@ -41,9 +48,6 @@ gcloud functions deploy "$name" --trigger-http \ --memory 128MB \ --region "$region" \ --timeout 60 \ + "${opts[@]}" \ --quiet # don't prompt to --allow-unauthenticated - # routes private traffic only by default - need to change this if you want to - # use the Cloud NAT gateway from the VPC for Firewall rules purposes - #--vpc-connector "$vpc_connector" \ - #--egress-settings all # sends all traffic through connector to exit via network's Cloud NAT IP #--max-instances 1 From 372749efe55533903a72c51a5452a78934cf7530 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:29:19 +0100 Subject: [PATCH 1788/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d36cef7cc..62a9153c7 100644 --- a/README.md +++ b/README.md @@ -467,7 +467,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu * [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. -* [Kubernetes templates](https://github.com/HariSekhon/Kubernetes-templates) - Kubernetes YAML templates - Best Practices, Tips & Tricks are baked right into the templates for future deployments +* [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments * [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) From ab3adcfe269c8c3c16ad5ebb7dcac14b8243fe82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:34:20 +0100 Subject: [PATCH 1789/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 528aa8999..38f6feaef 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 528aa89996f39b5d6c4d0df41404d9ca4c6fff3d +Subproject commit 38f6feaef03246160bf107fdf82ac84962b0555c From 79fc0e51188181498f567571a20f94abcb35bda9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:34:20 +0100 Subject: [PATCH 1790/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 964ff75d9..bbffddce9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 964ff75d90eb7337a998bbab4e832ca7307b4f04 +Subproject commit bbffddce93a5d9a934e4fa3ae21af2008e43061d From 538157988744146fa7545555753b213ed02a8cf7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:34:21 +0100 Subject: [PATCH 1791/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 80e2004a1..bdd9588ea 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 80e2004a17865c23992f3ad8d5bae3d55e60dc46 +Subproject commit bdd9588ea844c843cfaf2c054e485b85fe85bd82 From 5b992595ddc40240ddc2b47aaa771c3cfb6cee0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Jun 2021 10:34:21 +0100 Subject: [PATCH 1792/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 4775e138b..cd774ac45 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 4775e138bbb851f9157e318060f420389d3b695d +Subproject commit cd774ac45526f9fa0263136ffbf5176db7b0a756 From d40ee46f35922d334e1263ca9851337a841502d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 15:17:04 +0100 Subject: [PATCH 1793/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 38f6feaef..9bfa74658 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 38f6feaef03246160bf107fdf82ac84962b0555c +Subproject commit 9bfa74658bfd271e0912306e053ec5690fa834e1 From 99f1f60ead060c78ed80e18a73520c71f656802a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 15:17:05 +0100 Subject: [PATCH 1794/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index bbffddce9..70bd42751 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit bbffddce93a5d9a934e4fa3ae21af2008e43061d +Subproject commit 70bd427512cd2f862cf7b9622e8fa32b11726a75 From 424bbc81afe68c58789baf31aa9d74accdf246f9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 15:17:05 +0100 Subject: [PATCH 1795/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index bdd9588ea..b9c39d4e4 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit bdd9588ea844c843cfaf2c054e485b85fe85bd82 +Subproject commit b9c39d4e4e121a9f83f3a588df8bd17f94f57dde From 5e304641be41f88d527ac4c16e876572d81b90dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 15:17:05 +0100 Subject: [PATCH 1796/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index cd774ac45..994300130 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit cd774ac45526f9fa0263136ffbf5176db7b0a756 +Subproject commit 9943001302a2751781bf345584d100e6afcb5b07 From 77bd0b858d93b39d9bef1f24c3f35f0b26d0fdb1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 8 Jul 2021 17:37:22 +0100 Subject: [PATCH 1797/2779] updated Makefile --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) mode change 100755 => 100644 Makefile diff --git a/Makefile b/Makefile old mode 100755 new mode 100644 index abe77d452..66b95292f --- a/Makefile +++ b/Makefile @@ -78,8 +78,14 @@ init: # # dependency of same % stem prefix checks for a matching .py file to consider it a valid target # +# doesn't work +#.PHONY: all +#.PHONY: %.pyc +# TODO: doesn't work, says nothing to be done even when .pyc isn't present, and allows make anonymize22.py which breaks +#%.py: %.pyc +# @$(MAKE) $@c %.pyc:: %.py - @# this utility script supports taking .pyc or .pyo names and still does the right thing, + @# this utility script supports taking .pyc or .pyo names and still does the right thing @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ From cb6d31015052ea361bab2e4b19c3bcb56ac95966 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jul 2021 12:32:08 +0100 Subject: [PATCH 1798/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 62a9153c7..08255495b 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Hari Sekhon - DevOps Python Tools [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) -[![Travis CI](https://img.shields.io/travis/harisekhon/DevOps-Python-tools/master?logo=travis&label=Travis%20CI)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) +[![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) From 4253835d19bfe7ece79333793bbb8750fd606b7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Jul 2021 09:31:17 +0100 Subject: [PATCH 1799/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9bfa74658..743fda601 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9bfa74658bfd271e0912306e053ec5690fa834e1 +Subproject commit 743fda60144424b8926fbd1851d5f99209c09cc0 From cfd9e884a288da2e1b70155a99f1f34fc97a2db8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Jul 2021 09:31:18 +0100 Subject: [PATCH 1800/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 70bd42751..2c756632a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 70bd427512cd2f862cf7b9622e8fa32b11726a75 +Subproject commit 2c756632a83aad4b456f8f86d90de40f3c6622e4 From 40666b12c3fe7c618331ea13cbbf5fe43903c47e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Jul 2021 09:31:18 +0100 Subject: [PATCH 1801/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index b9c39d4e4..25e563ad7 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit b9c39d4e4e121a9f83f3a588df8bd17f94f57dde +Subproject commit 25e563ad72735fe868ed01534b3313322b68f49e From 61977ccfb5869b6c4691918930650da70cac0181 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 23 Jul 2021 09:31:18 +0100 Subject: [PATCH 1802/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 994300130..9c0c67b07 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9943001302a2751781bf345584d100e6afcb5b07 +Subproject commit 9c0c67b07d55aca66986d0d2b62c7952e72f6afc From 8e8f7d10de266d884e66a46978b7f4cd860c4c2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:34:24 +0100 Subject: [PATCH 1803/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 09b556190..6e5dc9f25 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -67,7 +67,7 @@ blocks: agent: machine: type: a1-standard-4 - os_image: macos-xcode11 + os_image: macos-xcode12 prologue: commands: - cache restore From 6e4f74da5f97aabbb2f32ba4a0621fa128f1833c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:50:00 +0100 Subject: [PATCH 1804/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 743fda601..14feb8065 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 743fda60144424b8926fbd1851d5f99209c09cc0 +Subproject commit 14feb80653901b7f59a4fc3b5ea8ca57d1e25eda From b36a00091c04c1155f112ad7fab580489a1c7fe4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:50:00 +0100 Subject: [PATCH 1805/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 2c756632a..bb608550d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 2c756632a83aad4b456f8f86d90de40f3c6622e4 +Subproject commit bb608550d7163fde875e0df9e31bacbfa77350e3 From 1dcfe0cd4be79b8088cc6f8395bd0e413163ca37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:50:00 +0100 Subject: [PATCH 1806/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 25e563ad7..d1989db29 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 25e563ad72735fe868ed01534b3313322b68f49e +Subproject commit d1989db297219f5fa0fa81f213a3a098df01f9fb From e386857bb0b97788938aeb8d741d287378557852 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 1 Oct 2021 10:50:00 +0100 Subject: [PATCH 1807/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9c0c67b07..c0a4f0054 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9c0c67b07d55aca66986d0d2b62c7952e72f6afc +Subproject commit c0a4f005460f04e104baa1c0d8095e4fdc687696 From 93dcd18b057d9dd93321d6c36ba456479626389d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Oct 2021 16:37:58 +0100 Subject: [PATCH 1808/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 14feb8065..248c4b862 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 14feb80653901b7f59a4fc3b5ea8ca57d1e25eda +Subproject commit 248c4b86270c680a38a70fc462bd35ff06dfbdb6 From 8010521b0399950ffe490acfa8cd6f545b989063 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 Nov 2021 19:08:51 +0000 Subject: [PATCH 1809/2779] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..d6aee201a --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '37 15 * * 4' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From a9678fbfbefcf34f575a5be2c997d441b925f21d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 19 Nov 2021 16:47:38 +0000 Subject: [PATCH 1810/2779] stripped quay.io from start if found for easier pasting --- quay_show_tags.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/quay_show_tags.py b/quay_show_tags.py index 5743be955..64a2167e6 100755 --- a/quay_show_tags.py +++ b/quay_show_tags.py @@ -30,6 +30,7 @@ #from __future__ import unicode_literals import os +import re import sys import traceback srcdir = os.path.abspath(os.path.dirname(__file__)) @@ -43,7 +44,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.2' +__version__ = '0.6.3' class QuayTags(DockerHubTags): @@ -61,7 +62,10 @@ def run(self): self.quiet = self.get_opt('quiet') if not self.quiet: print('\nQuay.io ', end='') + re_quay = re.compile('^quay.io/', re.I) for arg in self.args: + if re_quay.match(arg): + arg = re_quay.sub('', arg) self.print_tags(arg) From fbdaf03607876b6c547bca0386137ccf3998f67c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 18:41:32 +0000 Subject: [PATCH 1811/2779] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 08255495b..68a99b2d8 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ Hari Sekhon - DevOps Python Tools [![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) +[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) +[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 54d9a8388367bdd2ccf444a784b2f098d044558e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 18:47:57 +0000 Subject: [PATCH 1812/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 68a99b2d8..2c9968f30 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Hari Sekhon - DevOps Python Tools ================================= -[![Codacy](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) +[![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) [![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) [![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) From c13fb459ea57eb9080d3ffe2e40ce265835f062e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:15:35 +0000 Subject: [PATCH 1813/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c9968f30..d9150f343 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ All programs come with a ```--help``` switch which includes a program descriptio Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```. -### DevOps Python Tools - Inventory: +### DevOps Python Tools - Inventory - Linux: - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) From 5741cb779f688b2b2983fb683803c2ceb4dc73a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:18:20 +0000 Subject: [PATCH 1814/2779] updated README.md --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d9150f343..f0d4b1863 100644 --- a/README.md +++ b/README.md @@ -463,33 +463,33 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -* [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 550+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 550+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies -* [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery +- [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery -* [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. +- [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. -* [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments +- [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments -* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +- [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) -* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... +- [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... -* [HAProxy Configs](https://github.com/HariSekhon/HAProxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. +- [HAProxy Configs](https://github.com/HariSekhon/HAProxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. -* [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 +- [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 -* [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule +- [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule -* [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library +- [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: -* https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb +- https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb ### Stargazers over time From c3e3b8c4dd03835fb73136ef925703ecc34e69c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:36:09 +0000 Subject: [PATCH 1815/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 248c4b862..9e172499f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 248c4b86270c680a38a70fc462bd35ff06dfbdb6 +Subproject commit 9e172499fad04aff95a06af08750187adbc82293 From 6138988cce450a9b4d1585cfd342390fde3c8f3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:36:09 +0000 Subject: [PATCH 1816/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index bb608550d..ce9da2c5e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit bb608550d7163fde875e0df9e31bacbfa77350e3 +Subproject commit ce9da2c5ed3f1b8f4c275c3349a83055a46d3c41 From 9f1dd7789fe7eb2109cabf6806e493b58e3b69d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:36:09 +0000 Subject: [PATCH 1817/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index d1989db29..53c1d53b0 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit d1989db297219f5fa0fa81f213a3a098df01f9fb +Subproject commit 53c1d53b0ff15f8d9d0b7191b522a1b0b1d0bf20 From f889d4d7ac0fd5f9c29c21393f8aba1968242837 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Jan 2022 19:36:09 +0000 Subject: [PATCH 1818/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c0a4f0054..9021150df 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c0a4f005460f04e104baa1c0d8095e4fdc687696 +Subproject commit 9021150df83050e91ff5c72894de46305d0d54ce From c6831e3d589a98d61910d6e13c7de6d284dd74d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Jan 2022 09:54:42 +0000 Subject: [PATCH 1819/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9e172499f..46e0915c8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9e172499fad04aff95a06af08750187adbc82293 +Subproject commit 46e0915c89142fb4c8a9f79e23d63cba1b76862f From ae5fb27cf2b11e288e0a7ebaed960797f4ce0b66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Jan 2022 11:14:09 +0000 Subject: [PATCH 1820/2779] cache bust trigger test From 179e61581ff21dd644909f1f99045817181a2d6d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Jan 2022 11:50:06 +0000 Subject: [PATCH 1821/2779] cache bust trigger test From 57983094222a684cd3f69871ab6084a7a1447c5b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 11:39:29 +0000 Subject: [PATCH 1822/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 46e0915c8..0a37c9613 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 46e0915c89142fb4c8a9f79e23d63cba1b76862f +Subproject commit 0a37c96138270c50f69ae7abdb54a8288b8e9e36 From 1215ca2c4f6924430b9e8608d858098215591748 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 12:33:16 +0000 Subject: [PATCH 1823/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0a37c9613..4ab612bd4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0a37c96138270c50f69ae7abdb54a8288b8e9e36 +Subproject commit 4ab612bd40b71e17441026c88ab23e3448692e81 From 01f7ff613d498b1346dd0461c1a76c7072810f50 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 12:34:06 +0000 Subject: [PATCH 1824/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4ab612bd4..1db52c531 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4ab612bd40b71e17441026c88ab23e3448692e81 +Subproject commit 1db52c531988dd950c7b68a9ff5c1056b0f97b42 From d94853421163ee9f1879146220ff9a49f591dd16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 12:34:06 +0000 Subject: [PATCH 1825/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ce9da2c5e..206facb47 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ce9da2c5ed3f1b8f4c275c3349a83055a46d3c41 +Subproject commit 206facb472197bc91ceadb837979b187176e40b0 From 944df777149d8465d7dd0266dbbf5e5593504367 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 12:34:06 +0000 Subject: [PATCH 1826/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9021150df..ef18ba7c6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9021150df83050e91ff5c72894de46305d0d54ce +Subproject commit ef18ba7c6f7fb4c045fbe269dc5ab527155fecf0 From 1f1215b91250874c0cefaa25ca4bfaa482d8ef53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 13:37:48 +0000 Subject: [PATCH 1827/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1db52c531..fd5702ff2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1db52c531988dd950c7b68a9ff5c1056b0f97b42 +Subproject commit fd5702ff25dc9b9ea35701615186f41e2606a0d0 From 3486ee77987e41e6b9177d7ca07a54a5e08eb779 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 13:37:48 +0000 Subject: [PATCH 1828/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 206facb47..6747c6826 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 206facb472197bc91ceadb837979b187176e40b0 +Subproject commit 6747c6826ec494f97a1da066d1ad8ed9213fbed7 From 8e3b202a87aace9f6aada13ba84eaca27a79c163 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 16 Jan 2022 13:37:48 +0000 Subject: [PATCH 1829/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index ef18ba7c6..bb698209a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit ef18ba7c6f7fb4c045fbe269dc5ab527155fecf0 +Subproject commit bb698209ad1db770b9284ecd1d2e5eb948e4ce07 From f30bdafb0d69a424a21605e918079420bc477b6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:12:58 +0000 Subject: [PATCH 1830/2779] updated .gitmodules --- .gitmodules | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitmodules b/.gitmodules index f5871372d..2bb07ff73 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,6 +9,8 @@ [submodule "sql"] path = sql url = https://github.com/HariSekhon/SQL-scripts + branch = master [submodule "templates"] path = templates url = https://github.com/HariSekhon/Templates + branch = master From dcdc5062136eb9b866425ca33a23967bc8025f34 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:13:56 +0000 Subject: [PATCH 1831/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fd5702ff2..62fb7d954 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fd5702ff25dc9b9ea35701615186f41e2606a0d0 +Subproject commit 62fb7d95497cdfcdbfb935240a746e43bbf20502 From fd98dd1225f360a5b6eef455e87a135ec2d7a1c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:14:10 +0000 Subject: [PATCH 1832/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index bb698209a..0ef3948cd 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit bb698209ad1db770b9284ecd1d2e5eb948e4ce07 +Subproject commit 0ef3948cd44e56ea996e1de66e708abde7bd4174 From 0d3c5e219cadc09bc29cae2914b10f47ee97d3a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:18:51 +0000 Subject: [PATCH 1833/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 62fb7d954..d65d2027b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 62fb7d95497cdfcdbfb935240a746e43bbf20502 +Subproject commit d65d2027b883addebb8b839c2c7ce8b02f4b31dd From 6db5d0e8373e928fb4e7f272ab8928da98498824 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:42:20 +0000 Subject: [PATCH 1834/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d65d2027b..66bb42884 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d65d2027b883addebb8b839c2c7ce8b02f4b31dd +Subproject commit 66bb4288488e53bda65e249301a88f45fc86242c From 86d55f6d709a00cdb3aea9352aa2144841406fb0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 13:59:56 +0000 Subject: [PATCH 1835/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 66bb42884..c8671005d 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 66bb4288488e53bda65e249301a88f45fc86242c +Subproject commit c8671005d60a0e9574836b60526fbcaefef11c9e From 4604a12d84c268aae044424df96d7dbe8f88c44f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 15:10:04 +0000 Subject: [PATCH 1836/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c8671005d..baaf591a1 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c8671005d60a0e9574836b60526fbcaefef11c9e +Subproject commit baaf591a1be91850bce33172539215a7ded76e64 From f8217ecb198d79841a2e66fe44f5b8ec904523e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 16:32:53 +0000 Subject: [PATCH 1837/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index baaf591a1..fa6c84931 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit baaf591a1be91850bce33172539215a7ded76e64 +Subproject commit fa6c84931699372f2f6ec198ed5c13bd3cae4f64 From 90114cc75f11985b8d39db79f0090dc0c18ee714 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 16:56:18 +0000 Subject: [PATCH 1838/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fa6c84931..192b08c74 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fa6c84931699372f2f6ec198ed5c13bd3cae4f64 +Subproject commit 192b08c74d368b751f9f461a309a45dd884e57ec From 9af717fdef069798ec0bec42529b43562f1370b1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 17 Jan 2022 17:00:18 +0000 Subject: [PATCH 1839/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f0d4b1863..22edf4ff5 100644 --- a/README.md +++ b/README.md @@ -463,7 +463,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 550+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 700+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From acbdc9888acc312db3e6bb2c4f2e656587d4e818 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:36:52 +0000 Subject: [PATCH 1840/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 192b08c74..0ad91e913 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 192b08c74d368b751f9f461a309a45dd884e57ec +Subproject commit 0ad91e913edec71d613fb0187d1e24a029e1286f From aa937d84488a7794e26b35967fa917fc77861213 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:49:13 +0000 Subject: [PATCH 1841/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0ad91e913..ed73806a8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0ad91e913edec71d613fb0187d1e24a029e1286f +Subproject commit ed73806a8cccc8f09aaa2d15c1f1654b6d197c43 From 634189cfba4eccbe79b75c138c621d7a7c64c341 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:49:13 +0000 Subject: [PATCH 1842/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6747c6826..d8d872aec 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6747c6826ec494f97a1da066d1ad8ed9213fbed7 +Subproject commit d8d872aec71828b9ecf89e695f10f350ca576b41 From db58b88db0994a18eca633a44e9aded7a1c3158e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:49:13 +0000 Subject: [PATCH 1843/2779] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 53c1d53b0..ba7b03f51 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 53c1d53b0ff15f8d9d0b7191b522a1b0b1d0bf20 +Subproject commit ba7b03f51e9a5deb3f9c388e6cbaf1aa2ba6ce3a From fe0a417e7e6c83d82d8f391e727bbe1a06ea3018 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Jan 2022 10:49:13 +0000 Subject: [PATCH 1844/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 0ef3948cd..a6808f63e 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 0ef3948cd44e56ea996e1de66e708abde7bd4174 +Subproject commit a6808f63eea44b4956e49c66e940691ad2fe519b From b571b9c00604c9964b144c0a374b4691d0fa5fb2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 11:46:31 +0000 Subject: [PATCH 1845/2779] updated .appveyor.yml --- .appveyor.yml | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index eb43a0250..f23536e32 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -41,30 +41,30 @@ on_finish: - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi install: -# workaround for: -# Some packages could not be installed. This may mean that you have -# requested an impossible situation or if you are using the unstable -# distribution that some required packages have not yet been created -# or been moved out of Incoming. -# The following information may help to resolve the situation: -# -# The following packages have unmet dependencies: -# mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed -# E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. -# devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed -# make[2]: *** [apt-packages] Error 123 -# make[2]: Leaving directory '/home/appveyor/projects/pylib' -# devops-python-tools/Makefile.in:212: recipe for target 'system-packages' failed -# -# adding "|| :" to the end of these commands causes them to be silently ignored! -- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list -- sudo apt purge -qy --allow-change-held-packages mssql-server -# this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 -#- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages -- setup/ci_bootstrap.sh -- make + # workaround for: + # Some packages could not be installed. This may mean that you have + # requested an impossible situation or if you are using the unstable + # distribution that some required packages have not yet been created + # or been moved out of Incoming. + # The following information may help to resolve the situation: + # + # The following packages have unmet dependencies: + # mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed + # E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. + # devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed + # make[2]: *** [apt-packages] Error 123 + # make[2]: Leaving directory '/home/appveyor/projects/pylib' + # devops-python-tools/Makefile.in:212: recipe for target 'system-packages' failed + # + # adding "|| :" to the end of these commands causes them to be silently ignored! + - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list + - sudo apt purge -yq --allow-change-held-packages mssql-server + # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 + #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages + - setup/ci_bootstrap.sh + - make test_script: -- make test + - make test build: off From d271f6500b45a83c990f67d2c30fea4e7e31ccca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 11:46:50 +0000 Subject: [PATCH 1846/2779] updated .concourse.yml --- .concourse.yml | 76 +++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/.concourse.yml b/.concourse.yml index 2abb4ac17..9ed7da3e2 100644 --- a/.concourse.yml +++ b/.concourse.yml @@ -14,45 +14,45 @@ # resources: -- name: github - icon: github-circle - type: git - source: - uri: https://github.com/harisekhon/devops-python-tools - branch: master -#- name: daily -# type: time -# source: -# interval: 1d + - name: github + icon: github-circle + type: git + source: + uri: https://github.com/harisekhon/devops-python-tools + branch: master + #- name: daily + # type: time + # source: + # interval: 1d # https://concourse-ci.org/golang-library-example.html jobs: -- name: build - public: false - plan: - - get: github # from resource above - trigger: true - #version: every # build every git commit, default: latest - - task: build - config: - platform: linux - image_resource: - type: docker-image - source: - repository: ubuntu - tag: latest - inputs: - - name: github - path: code - params: - CONCOURSE: 1 - run: - path: /bin/bash - args: - - -c - - | - cd code && - setup/ci_bootstrap.sh && - make init && - make ci test + - name: build + public: false + plan: + - get: github # from resource above + trigger: true + #version: every # build every git commit, default: latest + - task: build + config: + platform: linux + image_resource: + type: docker-image + source: + repository: ubuntu + tag: latest + inputs: + - name: github + path: code + params: + CONCOURSE: 1 + run: + path: /bin/bash + args: + - -c + - | + cd code && + setup/ci_bootstrap.sh && + make init && + make ci test From 1318871cea2024612415948592cd5b98c526fe3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:11:14 +0000 Subject: [PATCH 1847/2779] updated .drone.yml --- .drone.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.drone.yml b/.drone.yml index beda97729..790bdcf4c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,3 +1,5 @@ +--- +# XXX: putting this separator further down with code causes a parsing bug in drone lint # # Author: Hari Sekhon # Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020) @@ -28,16 +30,16 @@ type: docker name: default steps: -- name: build - image: ubuntu:18.04 -# environment: -# DEBUG: 1 - commands: - - setup/ci_bootstrap.sh - - make init - - make ci - - make test + - name: build + image: ubuntu:18.04 + #environment: + # DEBUG: 1 + commands: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test trigger: branch: - - master + - master From fb3eab1a9369d6119098631c0c80d5007f119375 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:11:49 +0000 Subject: [PATCH 1848/2779] updated .gocd.yml --- .gocd.yml | 115 +++++++++++++++++++++++++++--------------------------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/.gocd.yml b/.gocd.yml index 54f7a3067..69d289bf4 100644 --- a/.gocd.yml +++ b/.gocd.yml @@ -16,6 +16,7 @@ # https://docs.gocd.org/current/configuration/configuration_reference.html +--- format_version: 3 pipelines: devops-python-tools: @@ -30,60 +31,60 @@ pipelines: auto_update: true branch: master stages: - - build-and-test: - fetch_materials: true - keep_artifacts: false - clean_workspace: false - approval: - type: success - allow_only_on_success: false - jobs: -# apt-update: -# timeout: 10 -# tasks: -# - exec: -# command: apt -# arguments: -# - update -# run_if: passed -# install-make: -# timeout: 10 -# tasks: -# - exec: -# command: apt -# arguments: -# - install -# - -qy -# - git -# - make -# run_if: passed - ci-bootstrap: - timeout: 10 - tasks: - - exec: - command: setup/ci_bootstrap.sh - run_if: passed - init: - timeout: 10 - tasks: - - exec: - command: make - arguments: - - init - run_if: passed - build: - timeout: 60 - tasks: - - exec: - command: make - arguments: - - ci - run_if: passed - test: - timeout: 60 - tasks: - - exec: - command: make - arguments: - - test - run_if: passed + - build-and-test: + fetch_materials: true + keep_artifacts: false + clean_workspace: false + approval: + type: success + allow_only_on_success: false + jobs: + #apt-update: + # timeout: 10 + # tasks: + # - exec: + # command: apt + # arguments: + # - update + # run_if: passed + #install-make: + # timeout: 10 + # tasks: + # - exec: + # command: apt + # arguments: + # - install + # - -qy + # - git + # - make + # run_if: passed + ci-bootstrap: + timeout: 10 + tasks: + - exec: + command: setup/ci_bootstrap.sh + run_if: passed + init: + timeout: 10 + tasks: + - exec: + command: make + arguments: + - init + run_if: passed + build: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - ci + run_if: passed + test: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - test + run_if: passed From 73982317e98531a54bd6d1ad86163b680c4e4e44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:12:11 +0000 Subject: [PATCH 1849/2779] updated azure-pipelines.yml --- azure-pipelines.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d8f0e09ac..50f6cccb2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,7 +16,7 @@ # https://aka.ms/yaml trigger: -- master + - master pool: # there is no /dev/stderr on this azure build! @@ -28,22 +28,22 @@ pool: #container: ubuntu:18.04 steps: -# requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' -- script: env | sort - displayName: env + # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' + - script: env | sort + displayName: env -# doesn't work in container due to unprivileged execution and lack of sudo -#- script: sudo apt-get update && sudo apt-get install -y git make -# displayName: install git & make + # doesn't work in container due to unprivileged execution and lack of sudo + #- script: sudo apt-get update && sudo apt-get install -y git make + # displayName: install git & make -#- script: make -# displayName: build + #- script: make + # displayName: build -# doesn't work in vmImage build due to lack of access to normal /dev/stderr device -# tee: /dev/stderr: No such device or address -#- script: make test -# displayName: test + # doesn't work in vmImage build due to lack of access to normal /dev/stderr device + # tee: /dev/stderr: No such device or address + #- script: make test + # displayName: test -# hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( -- script: sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c 'set -ex && cd /code && setup/ci_bootstrap.sh && make init && make ci test' - displayName: docker build + # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( + - script: sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c 'set -ex && cd /code && setup/ci_bootstrap.sh && make init && make ci test' + displayName: docker build From 81981b20d2debe8f9179e33c463d88b5583fef21 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:14:17 +0000 Subject: [PATCH 1850/2779] updated config.yml --- .circleci/config.yml | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 69f7a6179..80c4cfbb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,27 +14,32 @@ # https://www.linkedin.com/in/harisekhon # -# https://circleci.com/docs/2.0/configuration-reference +# Master Template with more advanced config: +# +# https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml + +# Reference: +# +# https://circleci.com/docs/2.0/configuration-reference version: 2.1 + +workflows: + version: 2 + workflow: + jobs: + - build + jobs: build: - # technically a docker image is a better choice than machine - # but we want to introduce some native environment variation - # between build systems in order to test the repo's build automation is robust - machine: - image: default - #image: ubuntu-1604:201903-01 - # set to an actual docker image when running locally using circle_ci_job.sh - # docker image must have git installed to do the checkout - # so using harisekhon/dev:ubuntu instead of base ubuntu image - #image: harisekhon/dev:ubuntu + docker: + - image: cimg/base:2021.04 + resource_class: medium steps: - checkout + - setup_remote_docker: + version: 20.10.11 - run: setup/ci_bootstrap.sh - run: make init - run: make - # to allow docker networking to work - - run: sudo sysctl net.ipv4.ip_forward=1 - - run: sudo service docker restart - run: make test From e6aa90e89dd74daf8a237c04479e514854421f44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:15:49 +0000 Subject: [PATCH 1851/2779] updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 80c4cfbb5..37332c73c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: build: docker: - image: cimg/base:2021.04 - resource_class: medium + resource_class: small steps: - checkout - setup_remote_docker: From 8c9ee8f44f659113d1e74616378f355f46713b33 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 12:23:03 +0000 Subject: [PATCH 1852/2779] updated .travis.yml --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0b4f9044..71348b0bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ # https://docs.travis-ci.com/user/customizing-the-build/ +--- version: ~> 1.0 # ============================ @@ -126,10 +127,10 @@ before_cache: cache: - pip - directories: - - $HOME/.cache - - $HOME/.cpan - - $HOME/.cpanm - - $HOME/.gem + - $HOME/.cache + - $HOME/.cpan + - $HOME/.cpanm + - $HOME/.gem # ============================================== # https://docs.travis-ci.com/user/job-lifecycle/ From ea9a351a3debdf52b6c9da7bf2914f47c9cf81c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 13:55:24 +0000 Subject: [PATCH 1853/2779] added validate.yaml --- .github/workflows/validate.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/validate.yaml diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml new file mode 100644 index 000000000..e7e6e2306 --- /dev/null +++ b/.github/workflows/validate.yaml @@ -0,0 +1,33 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +name: Validation + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 0 * * 1' + +jobs: + validate: + name: Validate + uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master From 3d27f42fb9d6cd593d3cbcb9e43b9d62054cc5c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:05:42 +0000 Subject: [PATCH 1854/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ed73806a8..429a989d4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ed73806a8cccc8f09aaa2d15c1f1654b6d197c43 +Subproject commit 429a989d41baa9926c00a7172cfa220927139575 From 5f3daceaa563fe7a18109f02ca0bde833f90575f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:05:42 +0000 Subject: [PATCH 1855/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d8d872aec..6b5c9b93e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d8d872aec71828b9ecf89e695f10f350ca576b41 +Subproject commit 6b5c9b93e107a321cbd1000aecbff96bf7fe952c From a9797205d5af51ed3819789d26537a83da10d8af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:05:43 +0000 Subject: [PATCH 1856/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a6808f63e..4893b0b33 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a6808f63eea44b4956e49c66e940691ad2fe519b +Subproject commit 4893b0b33b0d95ddc5549e8b58dcfd0d746d9d15 From 82f29b81f9183b30120b49953c501af24d5d6295 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:29:48 +0000 Subject: [PATCH 1857/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 429a989d4..a2c295fb4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 429a989d41baa9926c00a7172cfa220927139575 +Subproject commit a2c295fb4e04be4af767a69b70bcceeab164a2f7 From 66dafe2315dfb4d1d14f55df7590539ee909c401 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:35:56 +0000 Subject: [PATCH 1858/2779] updated compile.sh --- tests/compile.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/compile.sh b/tests/compile.sh index e235cfd70..952836a20 100755 --- a/tests/compile.sh +++ b/tests/compile.sh @@ -19,17 +19,18 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; -. ./tests/utils.sh +# shellcheck disable=SC1090 +. "$srcdir/utils.sh" hr echo "Compiling all Python files" hr echo -for x in $(find . -iname '*.py' -o -iname '*.jy'); do - isExcluded "$x" && continue - echo "compiling $x" - python -m py_compile $x -done +while read -r filename; do + isExcluded "$filename" && continue + echo "compiling $filename" + python -m py_compile "$filename" +done < <(find . -iname '*.py' -o -iname '*.jy') echo echo From 8e32bbd7f49e7f9593c73202574ac4cfe817a2cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:37:52 +0000 Subject: [PATCH 1859/2779] updated test_xml_to_json.sh --- tests/test_xml_to_json.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index 8d01e3750..abfd23a73 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -15,19 +15,19 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(cd "$(dirname "$0")" && pwd)" +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$srcdir"; +cd "$srcdir" -# shellcheck disable=SC1091 -. utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/utils.sh" -# shellcheck disable=SC1091 -. ../bash-tools/lib/utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/../bash-tools/lib/utils.sh" section "XML => JSON" -cd .. +cd "$srcdir/.." testdata="tests/data/simple.xml" From 2d3e6e29332627f1b17b99cb30831d1db917c270 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:38:43 +0000 Subject: [PATCH 1860/2779] updated test_xml_to_yaml.sh --- tests/test_xml_to_yaml.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_xml_to_yaml.sh b/tests/test_xml_to_yaml.sh index 6f93a6ddf..81167836d 100755 --- a/tests/test_xml_to_yaml.sh +++ b/tests/test_xml_to_yaml.sh @@ -15,19 +15,19 @@ set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(cd "$(dirname "$0")" && pwd)" +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$srcdir"; +cd "$srcdir" -# shellcheck disable=SC1091 -. utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/utils.sh" -# shellcheck disable=SC1091 -. ../bash-tools/lib/utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/../bash-tools/lib/utils.sh" section "XML => YAML" -cd .. +cd "$srcdir/.." testdata="tests/data/simple.xml" From 7d5a3d5cb3677f8a92db35be6984667e725ba0a1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:40:14 +0000 Subject: [PATCH 1861/2779] updated all.sh --- tests/all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/all.sh b/tests/all.sh index b1e1957dc..103094127 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -29,8 +29,9 @@ section "Running PyTools ALL" # runs against . by default cd "$srcdir/.."; +# shellcheck disable=SC1090 # has to be included so that isExcluded function is inherited -. bash-tools/check_all.sh +. "$srcdir/../bash-tools/check_all.sh" #tests/test_yamllint.sh From 89c8f5ea5b713bb9f2a956f69a496a87c081d6c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 14:44:42 +0000 Subject: [PATCH 1862/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 22edf4ff5..678fdf317 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ Hari Sekhon - DevOps Python Tools [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) -[![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) @@ -55,6 +54,8 @@ Hari Sekhon - DevOps Python Tools [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) +[![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) +[![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) From d4716aa6a1d9e9e7c3dab91e7e530d0940c4a46b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 15:03:50 +0000 Subject: [PATCH 1863/2779] updated .ipython-notebook-pyspark.00-pyspark-setup.py --- .ipython-notebook-pyspark.00-pyspark-setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ipython-notebook-pyspark.00-pyspark-setup.py b/.ipython-notebook-pyspark.00-pyspark-setup.py index 4dde897da..1de53b3fb 100644 --- a/.ipython-notebook-pyspark.00-pyspark-setup.py +++ b/.ipython-notebook-pyspark.00-pyspark-setup.py @@ -24,4 +24,4 @@ sys.path.insert(0, os.path.join(spark_home, 'python')) for lib in glob.glob(os.path.join(spark_home, 'python/lib/py4j-*-src.zip')): sys.path.insert(0, lib) -execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) +execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) # pylint: disable=undefined-variable From f3e1e2727ef4372f978dd6db4e34bb2bbfcc9226 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 15:35:52 +0000 Subject: [PATCH 1864/2779] updated requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dfef9fec9..c14c0ec49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ avro==1.8.1 # requires Python == 3.4, build in Makefile instead #avro-python3==1.9.0 -awscli==1.16.241 +# AWS CLIv1 is obsolete and doesn't support SSO - use CLIv2 - see https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/setup/install_aws_cli.sh +#awscli==1.16.241 #bitarray==0.8.1 #boto==2.49.0 boto3==1.10.37 From 50696dcbea0700c9fc6bf53f47c8d4c335a53826 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 18:37:21 +0000 Subject: [PATCH 1865/2779] updated validate.yaml --- .github/workflows/validate.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index e7e6e2306..81196da71 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -24,7 +24,6 @@ on: - main workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - cron: '0 0 * * 1' jobs: From d62ecad232086b6c2785eabb0141122711c15076 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Jan 2022 19:28:27 +0000 Subject: [PATCH 1866/2779] added semgrep.yaml --- .github/workflows/semgrep.yaml | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/semgrep.yaml diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml new file mode 100644 index 000000000..3b145d0d1 --- /dev/null +++ b/.github/workflows/semgrep.yaml @@ -0,0 +1,43 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# S e m g r e p G i t H u b W o r k f l o w +# ============================================================================ # + +# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions + +--- +name: Semgrep + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + semgrep: + name: Semgrep + uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + permissions: + actions: read + contents: read + security-events: write From 5a52ecf8835f81053d4fa1c9e9e0d7dc3f585d8a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:36:34 +0000 Subject: [PATCH 1867/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a2c295fb4..41ad03ead 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a2c295fb4e04be4af767a69b70bcceeab164a2f7 +Subproject commit 41ad03eadbb7c87a1d65f171ff1c09a7e905246a From 6c25c1581d9f84b05e4f2748d441aef9fbec9702 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:36:34 +0000 Subject: [PATCH 1868/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6b5c9b93e..40556d510 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6b5c9b93e107a321cbd1000aecbff96bf7fe952c +Subproject commit 40556d510e99fb25ca190376d278726c4a743a8e From 7ab1f1b803880bcd0d257c217d3ce5a936b3f5b7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:36:34 +0000 Subject: [PATCH 1869/2779] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ba7b03f51..c19c3de26 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ba7b03f51e9a5deb3f9c388e6cbaf1aa2ba6ce3a +Subproject commit c19c3de26a87c2885489cd3b383c526ffc2945d9 From 0f98399148d46c44bfe7310a9d981da54781590f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:36:34 +0000 Subject: [PATCH 1870/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 4893b0b33..9c48fbbb1 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 4893b0b33b0d95ddc5549e8b58dcfd0d746d9d15 +Subproject commit 9c48fbbb1235a0b43ddbb657673a7d0445e81743 From 5624548a68635bb1b735680a47bea1ce7d826151 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:51:20 +0000 Subject: [PATCH 1871/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 41ad03ead..a3172a206 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 41ad03eadbb7c87a1d65f171ff1c09a7e905246a +Subproject commit a3172a206422fa1a2626ef5c422d41729c11dcb7 From 177671824ce3f9fe1cdc56f6e8ee896d2c5f86c6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 20 Jan 2022 17:53:18 +0000 Subject: [PATCH 1872/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a3172a206..befa8f0bd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a3172a206422fa1a2626ef5c422d41729c11dcb7 +Subproject commit befa8f0bdf7f366e41bda38379fc7865d4769cda From 244a496e95dd75086b9f0752747d410001c6eb5e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Jan 2022 23:53:02 +0000 Subject: [PATCH 1873/2779] added semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/semgrep-cloud.yaml diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml new file mode 100644 index 000000000..df624fb33 --- /dev/null +++ b/.github/workflows/semgrep-cloud.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# S e m g r e p C l o u d W o r k f l o w +# ============================================================================ # + +# Logs results to https://semgrep.dev/ + +--- +name: Semgrep Cloud + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + semgrep: + name: Semgrep Cloud + uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master From 7be35071169ff96605baca0ddec79e3ce4de948e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Jan 2022 23:53:40 +0000 Subject: [PATCH 1874/2779] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 3b145d0d1..b319f2356 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -15,6 +15,8 @@ # S e m g r e p G i t H u b W o r k f l o w # ============================================================================ # +# Generates code scanning alerts in GitHub's Security tab -> Code scanning alerts + # https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions --- @@ -37,7 +39,3 @@ jobs: semgrep: name: Semgrep uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master - permissions: - actions: read - contents: read - security-events: write From ec70d959f23e080eadec7ef349347d5938006ecb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 21 Jan 2022 23:56:39 +0000 Subject: [PATCH 1875/2779] added checkov.yaml --- .github/workflows/checkov.yaml | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/checkov.yaml diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml new file mode 100644 index 000000000..90ae12ff5 --- /dev/null +++ b/.github/workflows/checkov.yaml @@ -0,0 +1,41 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/harisekhon/templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +# ============================================================================ # +# C h e c k o v G i t H u b W o r k f l o w +# ============================================================================ # + +# Static analysis of Terraform code - publishes report to GitHub Security tab + +# https://github.com/bridgecrewio/checkov-action + +--- +name: Checkov + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + checkov: + name: Checkov + uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master From 28e285f6b11e787dbcbc2b99ab0332e49bd93a96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:07:57 +0000 Subject: [PATCH 1876/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 678fdf317..2445c90db 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,6 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) -[![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) @@ -68,6 +67,8 @@ Hari Sekhon - DevOps Python Tools [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) +[![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) +[![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![GitHub Actions Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/GitHub%20Actions%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22) [![Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac%22) From 1fb2a04b416bfba1e7e8da08b9dcd97b771c5ae8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:16:05 +0000 Subject: [PATCH 1877/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index befa8f0bd..14c0dc05b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit befa8f0bdf7f366e41bda38379fc7865d4769cda +Subproject commit 14c0dc05b6fa1f41b2ccdc44eca76808ff34d893 From c6c5f8de71a28075d6bedd8e837e1ee952546ae9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:16:05 +0000 Subject: [PATCH 1878/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 40556d510..e6088b82c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 40556d510e99fb25ca190376d278726c4a743a8e +Subproject commit e6088b82c68e3edbb89270a388b5d840a66d2838 From 7ef841d48d9aa87cdc6b7b217fdc9f720033cde0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:16:05 +0000 Subject: [PATCH 1879/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index c19c3de26..0c8e17fc8 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit c19c3de26a87c2885489cd3b383c526ffc2945d9 +Subproject commit 0c8e17fc883761f43289b7fd834b842ef74f1d8b From 75cc86218273f99c23902ae2e832a179bd9c6445 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:16:05 +0000 Subject: [PATCH 1880/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9c48fbbb1..eab20c1ec 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9c48fbbb1235a0b43ddbb657673a7d0445e81743 +Subproject commit eab20c1ec387a76265c2b1e9997a79e6764ab555 From 200fe62138230ada4ecc099b1e81172592153bd4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:19:53 +0000 Subject: [PATCH 1881/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 6550283db..b38994a81 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Alpine -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: alpine - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apk add --no-cache git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Alpine + uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + #with: + # debug: 1 From 41b01f5d975ccb0a551828388a5009c0dbb91831 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:20:09 +0000 Subject: [PATCH 1882/2779] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 6c8eada80..dab96c4f0 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Alpine 3 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: alpine:3 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apk add --no-cache git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Alpine 3 + uses: HariSekhon/GitHub-Actions/.github/workflows/alpine3.yaml@master + #with: + # debug: 1 From dda0500f89eb221b555d6b205e4c014411fc5b29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:21:25 +0000 Subject: [PATCH 1883/2779] updated centos.yaml --- .github/workflows/centos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 5c214e8fd..494327e4b 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest container: centos env: - repo: devops-python-tools + repo: DevOps-Python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 From b76491185a7fc4d978bad71540669d2f7981a799 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Jan 2022 00:22:47 +0000 Subject: [PATCH 1884/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index bb3c76ec6..93b146910 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest container: centos:7 env: - repo: devops-python-tools + repo: DevOps-Python-tools steps: # untars repo in docker container so git submodule update fails #- uses: actions/checkout@v2 From 300f7bb3d954a8677ef4dbf05feb351b21ff8287 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:06:58 +0000 Subject: [PATCH 1885/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 14c0dc05b..388c71d13 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 14c0dc05b6fa1f41b2ccdc44eca76808ff34d893 +Subproject commit 388c71d13b511ce05230dd12b1602e1775e245c3 From 2ea8ebb0db695a09064d5229061e6e7b7e121ad4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:06:58 +0000 Subject: [PATCH 1886/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e6088b82c..bc4f4309c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e6088b82c68e3edbb89270a388b5d840a66d2838 +Subproject commit bc4f4309ca17c0a0d9de4f656b2fbde6bb261a3a From 27dbaebc6036ae0ffa1b3a23e6ebbd9ba1887814 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:06:59 +0000 Subject: [PATCH 1887/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index eab20c1ec..71efdf44a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit eab20c1ec387a76265c2b1e9997a79e6764ab555 +Subproject commit 71efdf44afd970b14c24013dc669d876eaf86417 From 3b051d54d578465a90b73b8499c2d660732036fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:15:50 +0000 Subject: [PATCH 1888/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 388c71d13..f807763cc 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 388c71d13b511ce05230dd12b1602e1775e245c3 +Subproject commit f807763cc7b6380588de492d0fe3f828a9e91b65 From 54dc246a7284dff767396edd81fb7b3ad7496ad1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:15:50 +0000 Subject: [PATCH 1889/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index bc4f4309c..a57363bf3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit bc4f4309ca17c0a0d9de4f656b2fbde6bb261a3a +Subproject commit a57363bf30e07af15d34364bcdafc960de4cf883 From a9f6519f54d29c2a23396e178ff7368444ce7bfc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 28 Jan 2022 15:15:51 +0000 Subject: [PATCH 1890/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 71efdf44a..8f35ff7ea 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 71efdf44afd970b14c24013dc669d876eaf86417 +Subproject commit 8f35ff7ea92527852dd4cdfe640958b2646ae001 From 6261545c7eefabeae1681e3a6faf1fea534d4dcf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 31 Jan 2022 18:09:49 +0000 Subject: [PATCH 1891/2779] updated codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 83 ++++++++++++--------------- 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d6aee201a..5e74ab863 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,22 +1,14 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# +--- name: "CodeQL" on: push: - branches: [ master ] + branches: + - master pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: + - master schedule: - cron: '37 15 * * 4' @@ -32,39 +24,40 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'python' ] + language: + - python # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://git.io/codeql-language-support steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # Command-line programs to run using the OS shell. + # https://git.io/JvXDl + + # If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 95157166937531203f7b5e4e36c4787d9c49bf67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 31 Jan 2022 18:10:55 +0000 Subject: [PATCH 1892/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f807763cc..19c99c629 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f807763cc7b6380588de492d0fe3f828a9e91b65 +Subproject commit 19c99c6298a5b2f4d6d099128cc9525c917f0a82 From ebe7a4b691a03134cb6f01271058fc33af0a01bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 31 Jan 2022 18:10:55 +0000 Subject: [PATCH 1893/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index a57363bf3..23bfcebb9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit a57363bf30e07af15d34364bcdafc960de4cf883 +Subproject commit 23bfcebb9d80f042beadc738c44154edd102752f From 5b3cda764fd4d8151078db27ce1881e8317b56be Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 31 Jan 2022 18:10:56 +0000 Subject: [PATCH 1894/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 8f35ff7ea..1a693b8a1 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 8f35ff7ea92527852dd4cdfe640958b2646ae001 +Subproject commit 1a693b8a192bec6c1fdf13e55bfe4bbf3d5d1ee4 From a7a8c68ac62ded941d271d5e33a322c7423bebfb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 1 Feb 2022 11:55:33 +0000 Subject: [PATCH 1895/2779] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index b319f2356..fcf2abe13 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,5 +37,9 @@ on: jobs: semgrep: - name: Semgrep + name: Semgrep GitHub uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + permission: + actions: read + contents: read + security-events: write From 2a402bfdf520c1c18139bcb0d4940ec18227ff42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:00:53 +0000 Subject: [PATCH 1896/2779] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index b319f2356..fcf2abe13 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,5 +37,9 @@ on: jobs: semgrep: - name: Semgrep + name: Semgrep GitHub uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + permission: + actions: read + contents: read + security-events: write From d97cc000ee941182b6185d0cf3ebde26eedf0b4d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:01:17 +0000 Subject: [PATCH 1897/2779] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index fcf2abe13..23a921d88 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -39,7 +39,7 @@ jobs: semgrep: name: Semgrep GitHub uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master - permission: + permissions: actions: read contents: read security-events: write From dba7169a083dcedb299f35b6cc8090ba60734c1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:05:15 +0000 Subject: [PATCH 1898/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 19c99c629..81d8e5d52 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 19c99c6298a5b2f4d6d099128cc9525c917f0a82 +Subproject commit 81d8e5d524995f05c4500f0a395edaba780b6f7e From ccb267a1bebe8ec83f02f55b7dce26f720b8aae5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:05:15 +0000 Subject: [PATCH 1899/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 23bfcebb9..baffc23ac 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 23bfcebb9d80f042beadc738c44154edd102752f +Subproject commit baffc23aca45c259debcafc1e0bc80d0b22d8345 From 5c09b1af0661af032bb68b73bf0585e31bb9d4dd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:05:15 +0000 Subject: [PATCH 1900/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 0c8e17fc8..5c79c102c 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 0c8e17fc883761f43289b7fd834b842ef74f1d8b +Subproject commit 5c79c102ca4aadfca314f9c098acb55d2b9bfbe7 From c9a29a5520326de018a595765990ed254475261b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 2 Feb 2022 21:05:15 +0000 Subject: [PATCH 1901/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 1a693b8a1..bdebc016d 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 1a693b8a192bec6c1fdf13e55bfe4bbf3d5d1ee4 +Subproject commit bdebc016d72178a97ef09c7593e5157e15d1f9d9 From 4e2c5aee63eb7068420e240b2a0ae685f5beab56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:06:46 +0000 Subject: [PATCH 1902/2779] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index fcf2abe13..23a921d88 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -39,7 +39,7 @@ jobs: semgrep: name: Semgrep GitHub uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master - permission: + permissions: actions: read contents: read security-events: write From 445507835f9f8592c9510ea86fd0cf9016dcea0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:08:03 +0000 Subject: [PATCH 1903/2779] updated bootstrap.sh --- setup/bootstrap.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 5383ae1f0..306e8ecb4 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -15,11 +15,11 @@ # Alpine / Wget: # -# wget https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh && sh bootstrap.sh +# wget -O- https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh | sh # # Curl: # -# curl https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh | sh +# curl https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh | sh set -eu [ -n "${DEBUG:-}" ] && set -x @@ -40,16 +40,24 @@ if [ "$(uname -s)" = Darwin ]; then elif [ "$(uname -s)" = Linux ]; then echo "Bootstrapping on Linux: $repo" if type apk >/dev/null 2>&1; then - $sudo apk --no-cache add bash git make curl + $sudo apk --no-cache add bash git make curl wget elif type apt-get >/dev/null 2>&1; then + if [ -n "${CI:-}" ]; then + export DEBIAN_FRONTEND=noninteractive + fi opts="" if [ -z "${PS1:-}" ]; then opts="-qq" fi $sudo apt-get update $opts - $sudo apt-get install $opts -y git make curl + $sudo apt-get install $opts -y git make curl wget --no-install-recommends elif type yum >/dev/null 2>&1; then - $sudo yum install -y git make curl + if grep -qi 'NAME=.*CentOS' /etc/*release; then + echo "CentOS EOL detected, replacing yum base URL to vault to re-enable package installs" + $sudo sed -i 's/^[[:space:]]*mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Linux-* + $sudo sed -i 's|^#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|' /etc/yum.repos.d/CentOS-Linux-* + fi + $sudo yum install -y git make curl wget else echo "Package Manager not found on Linux, cannot bootstrap" exit 1 From 64829ab4b40ded3fd2e7518565e360d7bebf1184 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:11:06 +0000 Subject: [PATCH 1904/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 81d8e5d52..36728e719 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 81d8e5d524995f05c4500f0a395edaba780b6f7e +Subproject commit 36728e7191402d862c58732a839bda2db604699c From c9960ebe9c6311b42a04ec7b03a9e0261b24df20 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:11:06 +0000 Subject: [PATCH 1905/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index baffc23ac..cf61b55f6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit baffc23aca45c259debcafc1e0bc80d0b22d8345 +Subproject commit cf61b55f6ae953eabafaff97a71e94b3983d21c1 From 1ad2ea765cef0bcded1fa20a5070036874fc2bc7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:11:07 +0000 Subject: [PATCH 1906/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 5c79c102c..2926bf822 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 5c79c102ca4aadfca314f9c098acb55d2b9bfbe7 +Subproject commit 2926bf822469f9a2d2bda6fb5705bf3b6ed52358 From ac05856fa6a2bf46e69d48c23f4d86f690093ea5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 3 Feb 2022 14:11:07 +0000 Subject: [PATCH 1907/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index bdebc016d..054e52071 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit bdebc016d72178a97ef09c7593e5157e15d1f9d9 +Subproject commit 054e52071c6e9b48e8603e7abb28ce7ae89d05cb From 5f8ea90955bc40e6ccd084522312bdb673509269 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 12:08:51 +0000 Subject: [PATCH 1908/2779] updated LICENSE --- LICENSE | 46 ++++------------------------------------------ 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/LICENSE b/LICENSE index 03b527051..4860c1cf5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,45 +1,7 @@ -======================================= -HARI SEKHON LICENSE Revision 2013112300 -======================================= +Copyright 2015 Hari Sekhon -Copyright (c) 2006 onwards, Hari Sekhon -All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -Redistribution and use in source and binary forms, with or without -modification, is permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -3. All advertising materials mentioning features or use of this software - must display the following acknowledgement: - This product includes software developed by Hari Sekhon. -4. Neither the name Hari Sekhon nor any affiliates may be used to endorse or - promote products derived from this software without specific prior written - permission. -5. Modifications may be released to the public only with prior written permission - from Hari Sekhon. Forking on GitHub is permitted for the purpose of creating - patch pull requests back to the original repository. -6. Private modifications may be made to suit requirements, but any modifications - to this work, whether publicly disclosed or not, must be sent back to - Hari Sekhon via GitHub (https://github.com/harisekhon/devops-python-tools) or - LinkedIn (https://www.linkedin.com/in/harisekhon) - and must come under this same license. Any such modifications may be - reincorporated for the improvement of this software. -7. This work may not be sold without prior written permission from Hari Sekhon -8. This license may change at any time and the latest revision supersedes - all prior revisions. -9. Alternative licensing must be agreed in writing with Hari Sekhon prior - to public availability. +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THIS SOFTWARE IS PROVIDED BY Hari Sekhon ''AS IS'' AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL Hari Sekhon OR ANY AFFILIATED BODY BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From 10457e5b14e2d761189ca5bada6cbf55a3ccfbd0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 15:52:32 +0000 Subject: [PATCH 1909/2779] updated checkov.yaml --- .github/workflows/checkov.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 90ae12ff5..a31ba133a 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -39,3 +39,7 @@ jobs: checkov: name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master + permissions: + actions: read + contents: read + security-events: write From 5c6837cf2344cb3119450e9c5f349a7ab121325a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 16:04:13 +0000 Subject: [PATCH 1910/2779] updated validate.yaml --- .github/workflows/validate.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 81196da71..cf901ed71 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -11,6 +11,7 @@ # https://www.linkedin.com/in/harisekhon # +--- name: Validation on: From 230ce8fdfd038b0637fb0355fa090db181047d3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 16:10:58 +0000 Subject: [PATCH 1911/2779] removed .github/workflows/centos6.yaml.disabled --- .github/workflows/centos6.yaml.disabled | 49 ------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/centos6.yaml.disabled diff --git a/.github/workflows/centos6.yaml.disabled b/.github/workflows/centos6.yaml.disabled deleted file mode 100644 index b78c9de3a..000000000 --- a/.github/workflows/centos6.yaml.disabled +++ /dev/null @@ -1,49 +0,0 @@ -# -# Author: Hari Sekhon -# Date: Tue Feb 4 09:53:28 2020 +0000 -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/harisekhon/devops-python-tools -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback -# -# https://www.linkedin.com/in/harisekhon -# - -# Not supporting RHEL6 any more because it doesn't have GNU parallel package - -name: CentOS 6 - -#env: -# DEBUG: 1 - -on: - push: - branches: - - master - schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' - -jobs: - build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: centos:6 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test From bd11ceed0dc87cea96b00e484e53c7f4a3faf858 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:17:49 +0000 Subject: [PATCH 1912/2779] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index df624fb33..d18460b45 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -37,3 +37,5 @@ jobs: semgrep: name: Semgrep Cloud uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master + secrets: + SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} From 6b2dad16f356bc9adb5ebd3443080313333ed138 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:18:56 +0000 Subject: [PATCH 1913/2779] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 23a921d88..af3635d78 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,7 +37,7 @@ on: jobs: semgrep: - name: Semgrep GitHub + name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master permissions: actions: read From 03118a276424c549ee221bba149e30e23354b4b0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:24 +0000 Subject: [PATCH 1914/2779] updated centos.yaml --- .github/workflows/centos.yaml | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 494327e4b..f4bf8241b 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -11,37 +11,20 @@ # https://www.linkedin.com/in/harisekhon # +--- name: CentOS -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: centos - env: - repo: DevOps-Python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: CentOS + uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + #with: + # debug: 1 From 882930ebfe12d704d45c3d99fc0933e336b29ae4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:24 +0000 Subject: [PATCH 1915/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 93b146910..3f8e5b283 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -11,37 +11,20 @@ # https://www.linkedin.com/in/harisekhon # +--- name: CentOS 7 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: centos:7 - env: - repo: DevOps-Python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: CentOS 7 + uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + #with: + # debug: 1 From 3f4a12e3b13617cf6f896ed1f8b4d615a7432798 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:25 +0000 Subject: [PATCH 1916/2779] updated centos8.yaml --- .github/workflows/centos8.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 070024988..255c2c66e 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: CentOS 8 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: centos:8 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: CentOS 8 + uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + #with: + # debug: 1 From 69947f9f254dcc665bb3216312886f9f28aa6608 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:25 +0000 Subject: [PATCH 1917/2779] updated debian.yaml --- .github/workflows/debian.yaml | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d729ae64e..a150bb2eb 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Debian -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: debian - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Debian + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + #with: + # debug: 1 From 63b5dbcb3b876b2763ea8954beeb0ed1d0a2a502 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:26 +0000 Subject: [PATCH 1918/2779] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 35 ++++++++------------------------ 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 454fe8bb2..68e161841 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Debian 10 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: debian:10 # -slim gets java install package conflicts - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Debian 10 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian10.yaml@master + #with: + # debug: 1 From 712cb22f3ddb09b22df69000a420215fa5b6eddf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:26 +0000 Subject: [PATCH 1919/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 46 ++++++------------------ 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 96673f75d..c2d8e7078 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -4,53 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # -name: CI Debian 6 +--- +name: Debian 6 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 10 - runs-on: ubuntu-latest - container: debian:6 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - echo VERSION: ; cat /etc/*release /etc/*version 2>/dev/null || : - apt-get update && - apt-get install -y git make - - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" - - name: build - run: | - cd "/tmp/$repo" && - make - - name: test - run: | - cd "/tmp/$repo" && - make test + name: Debian 6 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian6.yaml@master + #with: + # debug: 1 From d9f55c88c18f604f9cc29aed9a9b886368c8ca75 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:26 +0000 Subject: [PATCH 1920/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 46 ++++++------------------ 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 622715687..f6329c136 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -4,53 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # -name: CI Debian 7 +--- +name: Debian 7 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 10 - runs-on: ubuntu-latest - container: debian:7-slim - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: | - ls -l /.dockerenv - echo "pwd is $PWD" - cat /etc/*release - apt-get update && - apt-get install -y git make - - name: git clone - run: | - cd /tmp && - git clone "https://github.com/harisekhon/$repo" - - name: build - run: | - cd "/tmp/$repo" && - make - - name: test - run: | - cd "/tmp/$repo" && - make test + name: Debian 7 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian7.yaml@master + #with: + # debug: 1 From 360303edbed9b88627f2dd715f8575dbe7076cc7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:27 +0000 Subject: [PATCH 1921/2779] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 35 +++++++++------------------------ 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 99703fe57..fa99760a7 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Debian 8 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: debian:8 # -slim gets java install package conflicts - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Debian 8 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian8.yaml@master + #with: + # debug: 1 From 88bf873e28d04fb9c0c04f502137aa46bf0e790d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:27 +0000 Subject: [PATCH 1922/2779] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 35 +++++++++------------------------ 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index db7819b07..e58fd190b 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Debian 9 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: debian:9 # -slim gets java install package conflicts - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Debian 9 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian9.yaml@master + #with: + # debug: 1 From 9bbe1cf93dcdb39440869d56b274933eb99378b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:28 +0000 Subject: [PATCH 1923/2779] updated fedora.yaml --- .github/workflows/fedora.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index c7ca8aaae..0d13ef9b0 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Fedora -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: fedora - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: yum install -y git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Fedora + uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master + #with: + # debug: 1 From 2707ee95abdac3719f94e919f9178774a2c0d8bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:28 +0000 Subject: [PATCH 1924/2779] updated mac.yaml --- .github/workflows/mac.yaml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 02cd075cd..c2216dc2d 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Mac -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: brew update - run: which brew && brew update || echo - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Mac + uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master + #with: + # debug: 1 From 443e322d00fd214023cfa6f9466f26cf8f328705 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:29 +0000 Subject: [PATCH 1925/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 33 ++++++++------------------------ 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 392688e0a..ec1d7a6f8 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Mac 10.15 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: macos-10.15 - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: ~/Library/Caches/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: brew update - run: which brew && brew update || echo - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Mac 10.15 + uses: HariSekhon/GitHub-Actions/.github/workflows/mac10.15.yaml@master + #with: + # debug: 1 From a6ac95e17dfc62dab7244a273d5b2056c0fadee5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:29 +0000 Subject: [PATCH 1926/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 38 +++++++----------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index f605de0f2..152523c3b 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -13,41 +13,17 @@ name: PyPy 2 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [pypy2] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: PyPy2 + uses: HariSekhon/GitHub-Actions/.github/workflows/pypy2.yaml@master + #with: + # debug: 1 From 468faaa047d756597b2fede7e4459a7bc829ace8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:29 +0000 Subject: [PATCH 1927/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 38 +++++++----------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 1cb9ef4f7..280145926 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -13,41 +13,17 @@ name: PyPy 3 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [pypy3] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: PyPy3 + uses: HariSekhon/GitHub-Actions/.github/workflows/pypy3.yaml@master + #with: + # debug: 1 From 03c6cf189b6d1cc230cc0136e9b403877d7b5ca8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:30 +0000 Subject: [PATCH 1928/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 485e4859b..c57282060 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 2.7 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [2.7] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 2.7 + uses: HariSekhon/GitHub-Actions/.github/workflows/python2.7.yaml@master + #with: + # debug: 1 From 09849af56b0e76a06ed4f181065fc293fcd67b43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:30 +0000 Subject: [PATCH 1929/2779] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index 15fa37297..a8b945742 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 3.5 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [3.5] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 3.5 + uses: HariSekhon/GitHub-Actions/.github/workflows/python3.5.yaml@master + #with: + # debug: 1 From 70226d805c18f241f2f2b8f3d17c87aa40d2e3e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:30 +0000 Subject: [PATCH 1930/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 8eeb1a0e3..b9b3ab180 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 3.6 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [3.6] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 3.6 + uses: HariSekhon/GitHub-Actions/.github/workflows/python3.6.yaml@master + #with: + # debug: 1 From a74ef1b4d904106501c021ec0100ff6abb1cc3c9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:31 +0000 Subject: [PATCH 1931/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 5bc6252f7..57720b5c3 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 3.7 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [3.7] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 3.7 + uses: HariSekhon/GitHub-Actions/.github/workflows/python3.7.yaml@master + #with: + # debug: 1 From 2eb63a26fc41a2c3466eebd068630494f55a8a64 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:31 +0000 Subject: [PATCH 1932/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 39 +++++++------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 0764a94dc..2b7abf97e 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -4,50 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Python 3.8 -#env: -# DEBUG: 1 - on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3] - python-version: [3.8] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: Python 3.8 + uses: HariSekhon/GitHub-Actions/.github/workflows/python3.8.yaml@master + #with: + # debug: 1 From 86241a549e8fb3ebec8447679d2966c5ad44f4e2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:32 +0000 Subject: [PATCH 1933/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index cac47c8a3..204bf7572 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:latest - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + #with: + # debug: 1 From cd0b50a1639e7319a59c5aba0d1bc5f26cb36bfa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:32 +0000 Subject: [PATCH 1934/2779] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 8e267edb4..3e01a484e 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu 14.04 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:14.04 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu 14.04 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu14.yaml@master + #with: + # debug: 1 From fbd61be719f394b6bcbf33876cfcbd2253a01d30 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:33 +0000 Subject: [PATCH 1935/2779] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index d5db70c05..8184d3801 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu 16.04 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:16.04 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu 16.04 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu16.yaml@master + #with: + # debug: 1 From 00b0d4bb64d5dc40f8091d8d18cc8adcc9f88966 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:32:54 +0000 Subject: [PATCH 1936/2779] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 2bf3b18a7..7d8cf7e9c 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu 18.04 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:18.04 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu 18.04 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu18.yaml@master + #with: + # debug: 1 From 08853eccd4e7045ac2f8a89d72e03862e05de81a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:33:22 +0000 Subject: [PATCH 1937/2779] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index bc0c83aac..aa7c0e846 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -4,44 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: Ubuntu 20.04 -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - container: ubuntu:20.04 - env: - repo: devops-python-tools - steps: - # untars repo in docker container so git submodule update fails - #- uses: actions/checkout@v2 - - name: install git & make - run: apt-get update -qq && apt-get install -qy git make - - name: git clone - run: cd /tmp && git clone "https://github.com/harisekhon/$repo" - - name: init - run: cd "/tmp/$repo" && git submodule update --init --recursive - - name: build - run: cd "/tmp/$repo" && make ci - - name: test - run: cd "/tmp/$repo" && make test + name: Ubuntu 20.04 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu20.yaml@master + #with: + # debug: 1 From 1eeff6d3844828556254f7fb6c50c9e979c32676 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:33:23 +0000 Subject: [PATCH 1938/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 33 ++++++++-------------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 08b8f24a3..103fa566b 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -4,42 +4,27 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/harisekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # # https://www.linkedin.com/in/harisekhon # +--- name: GitHub Actions Ubuntu -#env: -# DEBUG: 1 - -on: # [push] +on: push: branches: - master + workflow_dispatch: schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 7 * * *' + - cron: '0 7 * * *' jobs: build: - #name: build - timeout-minutes: 60 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-devops-python-tools # ${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-devops-python-tools - - name: init - run: make init - - name: build - run: make ci - - name: test - run: make test + name: GitHub Ubuntu + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master + #with: + # debug: 1 From 3916d589775b46e3ba38044e7b6d4504e4a3b380 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:42:19 +0000 Subject: [PATCH 1939/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 36728e719..385f229d4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 36728e7191402d862c58732a839bda2db604699c +Subproject commit 385f229d40c19eaf7962577244f1967033c3dfbf From 4750f61a3971ab03dfaea4addcbab48ecc0a1be1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:42:19 +0000 Subject: [PATCH 1940/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index cf61b55f6..22ad03fa9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit cf61b55f6ae953eabafaff97a71e94b3983d21c1 +Subproject commit 22ad03fa9aa7627c3a8971c4f3515b067d124121 From 2202b676bee2d9ef9a08702161272c5d2408147b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:42:19 +0000 Subject: [PATCH 1941/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 2926bf822..731f3d458 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 2926bf822469f9a2d2bda6fb5705bf3b6ed52358 +Subproject commit 731f3d4583b49881c1a267874acfaa6cfa6eb7e0 From d96d5348a90ea8d0b0bfef6301b38b208e5e74b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Feb 2022 19:42:19 +0000 Subject: [PATCH 1942/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 054e52071..edb86249c 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 054e52071c6e9b48e8603e7abb28ce7ae89d05cb +Subproject commit edb86249ca9efcea597a9dfd072097a4536734e5 From ef6e3c679bc3aaf230a2bec6e37c68e2cb246ccd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:10 +0000 Subject: [PATCH 1943/2779] added dockerhub_pytools_alpine.yaml --- .../workflows/dockerhub_pytools_alpine.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_alpine.yaml diff --git a/.github/workflows/dockerhub_pytools_alpine.yaml b/.github/workflows/dockerhub_pytools_alpine.yaml new file mode 100644 index 000000000..ff2197707 --- /dev/null +++ b/.github/workflows/dockerhub_pytools_alpine.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (Alpine) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: alpine + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-alpine + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From 4b46521c07c2da96ff0a61a6f37310366f238741 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:10 +0000 Subject: [PATCH 1944/2779] added dockerhub_pytools_centos.yaml --- .../workflows/dockerhub_pytools_centos.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_centos.yaml diff --git a/.github/workflows/dockerhub_pytools_centos.yaml b/.github/workflows/dockerhub_pytools_centos.yaml new file mode 100644 index 000000000..06778bd22 --- /dev/null +++ b/.github/workflows/dockerhub_pytools_centos.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (CentOS) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: latest centos + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-centos + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From f681cac11224c027728dde5824e2497880a33a08 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:10 +0000 Subject: [PATCH 1945/2779] added dockerhub_pytools_debian.yaml --- .../workflows/dockerhub_pytools_debian.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_debian.yaml diff --git a/.github/workflows/dockerhub_pytools_debian.yaml b/.github/workflows/dockerhub_pytools_debian.yaml new file mode 100644 index 000000000..aa9da7e6a --- /dev/null +++ b/.github/workflows/dockerhub_pytools_debian.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (Debian) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: debian + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-debian + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From b851f84d1b22ea1a9ed597b79c7d11ed9b6755d9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:11 +0000 Subject: [PATCH 1946/2779] added dockerhub_pytools_fedora.yaml --- .../workflows/dockerhub_pytools_fedora.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_fedora.yaml diff --git a/.github/workflows/dockerhub_pytools_fedora.yaml b/.github/workflows/dockerhub_pytools_fedora.yaml new file mode 100644 index 000000000..3aa3c800f --- /dev/null +++ b/.github/workflows/dockerhub_pytools_fedora.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (Fedora) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: fedora + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-fedora + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From 11fbd0c11c0d5ce35bab6234b38fa731b7629ebb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:25:11 +0000 Subject: [PATCH 1947/2779] added dockerhub_pytools_ubuntu.yaml --- .../workflows/dockerhub_pytools_ubuntu.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/dockerhub_pytools_ubuntu.yaml diff --git a/.github/workflows/dockerhub_pytools_ubuntu.yaml b/.github/workflows/dockerhub_pytools_ubuntu.yaml new file mode 100644 index 000000000..d53930274 --- /dev/null +++ b/.github/workflows/dockerhub_pytools_ubuntu.yaml @@ -0,0 +1,39 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: DockerHub Build (Ubuntu) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo: harisekhon/pytools + tags: ubuntu latest + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-ubuntu + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From a9026344c8b7e174166b5ec1c53fb1881077525f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 18:26:31 +0000 Subject: [PATCH 1948/2779] added ghcr_python_ubuntu.yaml --- .github/workflows/ghcr_python_ubuntu.yaml | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/ghcr_python_ubuntu.yaml diff --git a/.github/workflows/ghcr_python_ubuntu.yaml b/.github/workflows/ghcr_python_ubuntu.yaml new file mode 100644 index 000000000..fdd5ad61b --- /dev/null +++ b/.github/workflows/ghcr_python_ubuntu.yaml @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-09 18:07:10 +0000 (Wed, 09 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: GHCR Build (Ubuntu) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build_ghcr.yaml@master + with: + image: pytools + tags: ubuntu latest + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-ubuntu + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write From 25e68b03e644a0b96831062c47e9f7bc17c7d5ce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 19:31:58 +0000 Subject: [PATCH 1949/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 385f229d4..018732364 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 385f229d40c19eaf7962577244f1967033c3dfbf +Subproject commit 018732364cb003fc7cb63287b0ca08f4aa401b42 From 0f7f46084fdee8fa55cf3e3014893139d1905c3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 19:31:58 +0000 Subject: [PATCH 1950/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 22ad03fa9..8c97585a3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 22ad03fa9aa7627c3a8971c4f3515b067d124121 +Subproject commit 8c97585a367eedb957a25de2ae37582bd8c6f32e From b85cda9a1994aa2b4392618035936a43db3343c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 19:31:58 +0000 Subject: [PATCH 1951/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 731f3d458..103a1543f 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 731f3d4583b49881c1a267874acfaa6cfa6eb7e0 +Subproject commit 103a1543fa73a0adf4f0af710fcf04d099500170 From 1f7788deaeb431263a7778a69e94c09223a81210 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Feb 2022 19:31:58 +0000 Subject: [PATCH 1952/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index edb86249c..542fac6c4 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit edb86249ca9efcea597a9dfd072097a4536734e5 +Subproject commit 542fac6c4e00adcb8b885cceb07b5dfee5361c05 From f0e52eaf57ab9fc3e88accfeae0df0c6f6ceb91c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 11:18:24 +0000 Subject: [PATCH 1953/2779] updated requirements.txt --- requirements.txt | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c14c0ec49..6060cbe23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,30 @@ avro==1.8.1 + # requires Python == 3.4, build in Makefile instead #avro-python3==1.9.0 + # AWS CLIv1 is obsolete and doesn't support SSO - use CLIv2 - see https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/setup/install_aws_cli.sh #awscli==1.16.241 #bitarray==0.8.1 + #boto==2.49.0 boto3==1.10.37 #cassandra-driver==3.6.0 dicttoxml==1.7.4 + # Elasticsearch library must match major version :( # for Elasticsearch 2.x #elasticsearch>=2.0.0,<3.0.0 # for Elasticsearch 1.x #elasticsearch>=1.0.0,<2.0.0 + # fails on requiring newer version of setuptools #Flask==0.10.1 GitPython==2.1.15 + # this GCP API is surprisingly awful, not using #google-api-python-client==1.11.0 + happybase==1.0.0 humanize==0.5.1 impyla==0.16.0 @@ -26,36 +33,49 @@ jinja2==2.11.3 ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 + # Python 3.5+ #numpy==1.17.2 numpy==1.16.5 + # requires pg_config to build from source #psycopg2==2.8.4 psycopg2-binary==2.8.4 + python-cson==1.0.9 psutil==5.7.0 + # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 #PyHive==0.6.1 + +# doesn't work with non-trivial code #PyInstaller==3.3.1 -python-ldap==3.2.0 + +# gcc compile error on Alpine, don't think this is used either +#python-ldap==3.2.0 + #python-jenkins==0.4.13 # pulled in automatically by snakebite[kerberos] #python-krbV==1.0.90 # needed by avro + python-snappy==0.5 sasl==0.2.1 sh==1.12.14 selenium==3.141.0 + # pulls in python-KrbV as a dependency which doesn't build on Mac any more # relies on python-krbV is unmaintained and unported to Python 3 # - moved to Makefile as best effort #snakebite[kerberos]==2.11.0 #snakebite==2.11.0 + thrift-sasl==0.2.1 thrift==0.9.3 thriftpy==0.3.9 toml==0.10.0 xmltodict==0.10.2 yamllint==1.15.0 + #pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability. update: this breaks Python 3.5 build where this requirement is not found From cf4afe6858e23005b9c50a0fa6f34e0803909eda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:06 +0000 Subject: [PATCH 1954/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 152523c3b..61d7b8925 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -24,6 +24,7 @@ on: jobs: build: name: PyPy2 - uses: HariSekhon/GitHub-Actions/.github/workflows/pypy2.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: pypy2 + #debug: 1 From c6d05ecb8e505cfc1226e27e1bb3bfb2d8e3fce9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:07 +0000 Subject: [PATCH 1955/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 280145926..1b8cb30ed 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -24,6 +24,7 @@ on: jobs: build: name: PyPy3 - uses: HariSekhon/GitHub-Actions/.github/workflows/pypy3.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: pypy3 + #debug: 1 From 6169e0ea374df4b5249d7633252e178bf3d18744 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:07 +0000 Subject: [PATCH 1956/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index c57282060..8fc49fa55 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,7 @@ on: jobs: build: name: Python 2.7 - uses: HariSekhon/GitHub-Actions/.github/workflows/python2.7.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 2.7 + #debug: 1 From cd2f98dec3d3cbebb4a358dcd659c563f5b7546a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:08 +0000 Subject: [PATCH 1957/2779] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index a8b945742..f77c1fc28 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,7 @@ on: jobs: build: name: Python 3.5 - uses: HariSekhon/GitHub-Actions/.github/workflows/python3.5.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 3.5 + #debug: 1 From 8ff9c649f169bd195affd96ec3448898784e7774 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:08 +0000 Subject: [PATCH 1958/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index b9b3ab180..a661b9493 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,9 @@ on: jobs: build: name: Python 3.6 - uses: HariSekhon/GitHub-Actions/.github/workflows/python3.6.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master #with: # debug: 1 + with: + version: 3.5 + #debug: 1 From 6c2f481ccb08abd79ce7a0a4389c100aaa2b33e0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:09 +0000 Subject: [PATCH 1959/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 57720b5c3..c621468f0 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,7 @@ on: jobs: build: name: Python 3.7 - uses: HariSekhon/GitHub-Actions/.github/workflows/python3.7.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 3.7 + #debug: 1 From feff67e17eb584476d459542bbbcdf2eccb64b90 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:02:09 +0000 Subject: [PATCH 1960/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 2b7abf97e..636d06d06 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # @@ -25,6 +25,7 @@ on: jobs: build: name: Python 3.8 - uses: HariSekhon/GitHub-Actions/.github/workflows/python3.8.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 3.8 + #debug: 1 From 97fe5e5993d895b946f8c100518f05fbefb59791 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:09 +0000 Subject: [PATCH 1961/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index b38994a81..a0a127330 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -26,5 +26,6 @@ jobs: build: name: Alpine uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From 9284c551ca19ef799dad9f46d1beb426a7fd9085 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:10 +0000 Subject: [PATCH 1962/2779] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index dab96c4f0..409fd6f68 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Alpine 3 - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine3.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + with: + version: 3 + #debug: 1 From 71b2976aca14b23ad3347a780fe49a1617a7d773 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:10 +0000 Subject: [PATCH 1963/2779] updated centos.yaml --- .github/workflows/centos.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index f4bf8241b..10e7cb54a 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -26,5 +26,6 @@ jobs: build: name: CentOS uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From 5b03adef213117c1cf3fd33ffc14b73d72b61a25 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:11 +0000 Subject: [PATCH 1964/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 3f8e5b283..ec6f639b1 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -26,5 +26,6 @@ jobs: build: name: CentOS 7 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master - #with: - # debug: 1 + with: + version: 7 + #debug: 1 From 4d52f5b3a9ab19cabc47bcdb6a4ba13b95898eae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:11 +0000 Subject: [PATCH 1965/2779] updated centos8.yaml --- .github/workflows/centos8.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 255c2c66e..713ea3e67 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -26,5 +26,6 @@ jobs: build: name: CentOS 8 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master - #with: - # debug: 1 + with: + version: 8 + #debug: 1 From ec3ea8273742473f527380fc46ae733f140a461d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:12 +0000 Subject: [PATCH 1966/2779] updated debian.yaml --- .github/workflows/debian.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index a150bb2eb..78fc934f5 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -26,5 +26,6 @@ jobs: build: name: Debian uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From 1a8d7666cd2beed752fb62216ead8cf83c20c940 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:12 +0000 Subject: [PATCH 1967/2779] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 68e161841..ce950e6e8 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 10 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian10.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 10 + #debug: 1 From 661e8a8978367f2ca6fee6d28bc11f529ae5b066 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:13 +0000 Subject: [PATCH 1968/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index c2d8e7078..f087db959 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 6 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian6.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 6 + #debug: 1 From e5b1c8b5e408089bf27342c9fd0f8c877dfe6e25 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:13 +0000 Subject: [PATCH 1969/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index f6329c136..30677854f 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian7.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 7 + #debug: 1 From 0573d4c6038dd7440c29b855df3a2710162a511d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:14 +0000 Subject: [PATCH 1970/2779] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index fa99760a7..179419b4f 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian8.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 8 + #debug: 1 From f57cd0a7f5efb34605b24b5dca236d5d1e30591c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:14 +0000 Subject: [PATCH 1971/2779] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index e58fd190b..38e677a76 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Debian 9 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian9.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + with: + version: 9 + #debug: 1 From 620d29e47e879b65964cdeb7f31c305271918c2c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:14 +0000 Subject: [PATCH 1972/2779] updated mac.yaml --- .github/workflows/mac.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index c2216dc2d..0c9454bd6 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -26,5 +26,6 @@ jobs: build: name: Mac uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From b362f0c5dc86ad9db86792a97f8e040a96b065ed Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:15 +0000 Subject: [PATCH 1973/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index ec1d7a6f8..e214adb9a 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Mac 10.15 - uses: HariSekhon/GitHub-Actions/.github/workflows/mac10.15.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master + with: + version: 10.15 + #debug: 1 From f88475880eefad06c727d8902e0bf294fd7d0b44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:16 +0000 Subject: [PATCH 1974/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index a661b9493..623f85126 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -29,5 +29,5 @@ jobs: #with: # debug: 1 with: - version: 3.5 + version: 3.6 #debug: 1 From 151252eade057c51b1bfda7c681d86d41f438001 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:16 +0000 Subject: [PATCH 1975/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 204bf7572..d5c6df777 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -26,5 +26,6 @@ jobs: build: name: Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master - #with: - # debug: 1 + with: + version: latest + #debug: 1 From 3a5a85e06a26db6e2c7b8d8f8b910eb62738f921 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:17 +0000 Subject: [PATCH 1976/2779] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 3e01a484e..cda82b285 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Ubuntu 14.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu14.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + with: + version: 14.04 + #debug: 1 From 1523c810c31b295ae00960b8930d12743dbc7210 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:17 +0000 Subject: [PATCH 1977/2779] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 8184d3801..96b935a17 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Ubuntu 16.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu16.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + with: + version: 16.04 + #debug: 1 From 03fae4d5a2e26732bc0683270be6da632644e965 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:18 +0000 Subject: [PATCH 1978/2779] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 7d8cf7e9c..0c7f11c41 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Ubuntu 18.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu18.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + with: + version: 18.04 + #debug: 1 From ecc078d643cc40ccb6ed3c3edd28cb964b71d38b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:27:18 +0000 Subject: [PATCH 1979/2779] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index aa7c0e846..56701d844 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -25,6 +25,7 @@ on: jobs: build: name: Ubuntu 20.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu20.yaml@master - #with: - # debug: 1 + uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + with: + version: 20.04 + #debug: 1 From b1cfd31ed7b749c58b6bd881f7551c7457ac35ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:47 +0000 Subject: [PATCH 1980/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a0a127330..1b72eba8d 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Alpine From e72b4d7a4f3646026fea0fdf1821ccc923af7d5e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:48 +0000 Subject: [PATCH 1981/2779] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 409fd6f68..f041be5d2 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Alpine 3 From 658cddd6bed7feecf08f12b267f469d38bb943fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:48 +0000 Subject: [PATCH 1982/2779] updated centos.yaml --- .github/workflows/centos.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 10e7cb54a..5949b16cf 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: CentOS From 478e890aff50ed02b081c01cdac42d8e883048c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:48 +0000 Subject: [PATCH 1983/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index ec6f639b1..77815b813 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: CentOS 7 From 0046681a0849112f5a34fcd0912512b4ffa0c2c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:49 +0000 Subject: [PATCH 1984/2779] updated centos8.yaml --- .github/workflows/centos8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 713ea3e67..4df3a4cba 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: CentOS 8 From 83bf8fa04a09067df1c7999fa5e307ab8f2e4daa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:49 +0000 Subject: [PATCH 1985/2779] updated debian.yaml --- .github/workflows/debian.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 78fc934f5..13418a6ab 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian From 7383d246bea930dc9599839cbe17732b937aefe9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:49 +0000 Subject: [PATCH 1986/2779] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index ce950e6e8..8b5c32f96 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 10 From 6f8d28ec9e54820ec4179a85cb92fa6f9d41789f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:50 +0000 Subject: [PATCH 1987/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index f087db959..5023bbe64 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 6 From 14dcfb70f54abbed925da137abd3f7376d407b8b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:50 +0000 Subject: [PATCH 1988/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 30677854f..232c2893a 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 7 From c4ab9e1ca04bd5e7d518acce84d8cd21f56a33f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:50 +0000 Subject: [PATCH 1989/2779] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 179419b4f..65bb383ba 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 8 From d67da6d7a5af06423b34835d1816b02306075943 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:51 +0000 Subject: [PATCH 1990/2779] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 38e677a76..5308750e8 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Debian 9 From 55f744986525aef20bf3adcb57624bb5b74d4e61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:51 +0000 Subject: [PATCH 1991/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 61d7b8925..52c08058b 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -21,6 +21,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: PyPy2 From a8cb330177767d6023e06c16949576edf5fa5c39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:51 +0000 Subject: [PATCH 1992/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 1b8cb30ed..62afbfb46 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -21,6 +21,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: PyPy3 From 645f274ea43cdff7a81a78f81a03a21694eb7339 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:52 +0000 Subject: [PATCH 1993/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 8fc49fa55..8381f774f 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 2.7 From b8237f82369ddef17eb56d68b4ab565dd8bd3590 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:52 +0000 Subject: [PATCH 1994/2779] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index f77c1fc28..b735cfd0c 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 3.5 From eb105a91052e39e63202dcb496a9a8dd785d1b0e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:52 +0000 Subject: [PATCH 1995/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 623f85126..3cedc3c2c 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 3.6 From 6d08d020ba9bdf257731c25bc6f07b38306f42f5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:53 +0000 Subject: [PATCH 1996/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index c621468f0..1414475c1 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 3.7 From 72b852a90e505f5bc967904050c049de4e9326af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:53 +0000 Subject: [PATCH 1997/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 636d06d06..14bf646cd 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Python 3.8 From 142e19864a57032f98390e55fa5c902017db45d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:54 +0000 Subject: [PATCH 1998/2779] updated mac.yaml --- .github/workflows/mac.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 0c9454bd6..7224f71c2 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Mac From 94ac28ca6936a6817fd312ac2735a30e4e50ede2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:54 +0000 Subject: [PATCH 1999/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index e214adb9a..8753027af 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Mac 10.15 From 4515aac87bbf060dfa23657c3170b89aaa161b66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:54 +0000 Subject: [PATCH 2000/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index d5c6df777..5768ae6e7 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu From 0276f66b7797aaae4e5855e0a09ffc9b8da2bda5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:55 +0000 Subject: [PATCH 2001/2779] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index cda82b285..428f23491 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu 14.04 From 54745a99f7ccd298fd902bd5181fb548be4cdce7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:55 +0000 Subject: [PATCH 2002/2779] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 96b935a17..b61a9202d 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu 16.04 From b050e50986ae4f738ccd534b9c75ff25c1251941 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:55 +0000 Subject: [PATCH 2003/2779] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 0c7f11c41..49c5cf96c 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu 18.04 From 5cfdea1deb89c07d88041a129d3b87fcad993c13 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:56 +0000 Subject: [PATCH 2004/2779] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 56701d844..41d7d9d3f 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Ubuntu 20.04 From a101b530eee066e481f277fb9bfa70bbd3306349 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:34:56 +0000 Subject: [PATCH 2005/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 103fa566b..be5a946d8 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: GitHub Ubuntu From bc7a188b8d2f555fdb093fea22276c9860235f54 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 12:36:56 +0000 Subject: [PATCH 2006/2779] updated fedora.yaml --- .github/workflows/fedora.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 0d13ef9b0..99b3e8f79 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -22,6 +22,10 @@ on: schedule: - cron: '0 7 * * *' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + jobs: build: name: Fedora From 7e3c85a3d2e1d4d0dc9f4890b314ae6dada1cbef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:18 +0000 Subject: [PATCH 2007/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 1b72eba8d..a92204784 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 9ad486a7b40fabe657f642ff050dfc23459fd6a0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:18 +0000 Subject: [PATCH 2008/2779] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index f041be5d2..4b0cb4170 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 71888452fd4ce66f778ac9816adbd0a7a81c07c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2009/2779] updated centos.yaml --- .github/workflows/centos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 5949b16cf..ff1e33598 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 0e4b8776c6dd48cb5c277aacc7fc34b790aef626 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2010/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 77815b813..588f5de59 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From eff5e66fe0406f2ee4aa17f61471c4a260d0296c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2011/2779] updated centos8.yaml --- .github/workflows/centos8.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 4df3a4cba..23aeb1c79 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From b36e324ec7982d45d664269c5a1f4c4cef055322 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2012/2779] updated debian.yaml --- .github/workflows/debian.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 13418a6ab..0ac7e1f3e 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 0da43423e5c05179ad938e28b8aaeb79de50bf2e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2013/2779] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 8b5c32f96..7763b32f2 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 3623d46a49026f7c073e54f3ec26f46b112e60f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2014/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 5023bbe64..99015df89 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From e4ad04e9336418aa45c3f6bbc853406581d057af Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2015/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 232c2893a..3dea635f2 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 13fda2a38079378cc47156c3618ccac1737d979d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2016/2779] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 65bb383ba..8d043b023 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 48f562024119d487ec55d74a76a32e40cb2531ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2017/2779] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 5308750e8..6a9c34723 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 12904c3ff136c9ec52890cf16943282f206b429f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2018/2779] updated fedora.yaml --- .github/workflows/fedora.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 99b3e8f79..c893bc2cf 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 1879a606cd8836dcbb98f325a9a87ac39ed64ae5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2019/2779] updated mac.yaml --- .github/workflows/mac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 7224f71c2..ea6c12828 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From af632352f89908ef440908ca63569ad5525e2d0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2020/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 8753027af..b5c50bc4c 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 73c8a1ec8063f5a53846da89294cc2f7a5f25de4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2021/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 52c08058b..489d1bcc4 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -23,7 +23,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 249928a26f492a9a4f611f0d474bbafbfb61760a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:19 +0000 Subject: [PATCH 2022/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 62afbfb46..5c0e12095 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -23,7 +23,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 27d3c1170459d081081675da55cb5707fac900ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2023/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 8381f774f..c58d3aaa9 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From bbcf69a3380403d97d99c74331871d2b4c811a80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2024/2779] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index b735cfd0c..659bbc6f6 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 255b2e110bd07e67a9912507db2d5ab66adfb09f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2025/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 3cedc3c2c..b325c9523 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 504a7fd05586cbd767996eef43c4eb85440a10e0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2026/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 1414475c1..413e03916 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 9d58ee7f5a8580e3b87db085929994bc73124ee4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2027/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 14bf646cd..9621587bb 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From af83242bba9e9d4ee30fb1bd20fa186db391054d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2028/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 5768ae6e7..07c4af459 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 04f01563b59622c7da5ca4d275f51f83570eb540 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2029/2779] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 428f23491..561d7cfda 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From b17c666e25d841628123c0c58479bbffc9ac5c96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2030/2779] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index b61a9202d..59d45e0ae 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 2fbd28187fcb209637d067315f4e4f3c9126062a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2031/2779] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 49c5cf96c..d3818d27d 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 996ea9a0b800e4fc974d3fc25f8385e3e5b57ed0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2032/2779] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 41d7d9d3f..7ceee32d7 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 79c9b6a7f4840eb15f24ce8e07768eaea360e314 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:15:20 +0000 Subject: [PATCH 2033/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index be5a946d8..4c463496a 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -24,7 +24,7 @@ on: concurrency: group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: false # build auto-cancellation - enabling this causes false-positive badge failure statuses + cancel-in-progress: true jobs: build: From 3b37de4bff5f37abf98f65b9d51536b568e4fe74 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:27:16 +0000 Subject: [PATCH 2034/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 018732364..e71685633 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 018732364cb003fc7cb63287b0ca08f4aa401b42 +Subproject commit e71685633a9bf9d8aa19dc9f9597194e4ce5eae4 From 2269bb68cf2f48e114f1ef3a9654de697f87ab6d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:27:16 +0000 Subject: [PATCH 2035/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8c97585a3..4fa30554f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8c97585a367eedb957a25de2ae37582bd8c6f32e +Subproject commit 4fa30554fc746d5eb90e374c7782bc1bef7f6c47 From 493f0d25b80d28fdbeb32c00f7595bf0bd7a6e83 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Feb 2022 14:27:16 +0000 Subject: [PATCH 2036/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 542fac6c4..9b14e6b9b 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 542fac6c4e00adcb8b885cceb07b5dfee5361c05 +Subproject commit 9b14e6b9be41fd29ba76a585bd490721be05b2d6 From 1520679f301aa861d16c8b709d63ed4a90413668 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 16 Feb 2022 17:29:09 +0000 Subject: [PATCH 2037/2779] added CODEOWNERS --- .github/CODEOWNERS | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..636853854 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,20 @@ +# +# Author: Hari Sekhon +# Date: 2021-11-09 15:14:59 +0000 (Tue, 09 Nov 2021) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +# Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam + +#* @harisekhon From 17475b8329b4be392ed748bd9014ecdff5b77392 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 11:12:46 +0000 Subject: [PATCH 2038/2779] added fork-update.yaml --- .github/workflows/fork-update.yaml | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/fork-update.yaml diff --git a/.github/workflows/fork-update.yaml b/.github/workflows/fork-update.yaml new file mode 100644 index 000000000..7239ea515 --- /dev/null +++ b/.github/workflows/fork-update.yaml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: Fork Update + +on: + workflow_dispatch: + inputs: + debug: + type: string + required: false + schedule: + - cron: '0 10 * * 2' + +jobs: + fork_update: + name: Fork Update + uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update.yaml@master + with: + debug: ${{ github.event.inputs.debug }} + permissions: + contents: write + pull-requests: write From d54611586c5524480da75511b609da6d3bf637ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:49:09 +0000 Subject: [PATCH 2039/2779] added fork-sync.yaml --- .github/workflows/fork-sync.yaml | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/fork-sync.yaml diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml new file mode 100644 index 000000000..aa17a532e --- /dev/null +++ b/.github/workflows/fork-sync.yaml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: Fork Sync + +on: + workflow_dispatch: + inputs: + debug: + type: string + required: false + schedule: + - cron: '0 * * * *' + +jobs: + fork_sync: + if: github.repository_owner != 'HariSekhon' + name: Fork Sync + uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master + with: + debug: ${{ github.event.inputs.debug }} + permissions: + contents: write From 865e37884ce7472f216a015fcd650811ed28560b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:49:10 +0000 Subject: [PATCH 2040/2779] added fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/fork-update-pr.yaml diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml new file mode 100644 index 000000000..66315c238 --- /dev/null +++ b/.github/workflows/fork-update-pr.yaml @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/harisekhon +# + +--- +name: Fork Update PR + +on: + workflow_dispatch: + inputs: + debug: + type: string + required: false + schedule: + # fork-sync happens for default branch every hour, so just after that, run PRs for branches + - cron: '2 10 * * 2' + +jobs: + fork_update_pr: + if: github.repository_owner != 'HariSekhon' + name: Fork Update PR + uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master + with: + debug: ${{ github.event.inputs.debug }} + permissions: + contents: write + pull-requests: write From 29b997ba45497e1726f6497ebfbbe56494e07490 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:49:37 +0000 Subject: [PATCH 2041/2779] removed .github/workflows/fork-update.yaml --- .github/workflows/fork-update.yaml | 34 ------------------------------ 1 file changed, 34 deletions(-) delete mode 100644 .github/workflows/fork-update.yaml diff --git a/.github/workflows/fork-update.yaml b/.github/workflows/fork-update.yaml deleted file mode 100644 index 7239ea515..000000000 --- a/.github/workflows/fork-update.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# -# Author: Hari Sekhon -# Date: Tue Feb 4 09:53:28 2020 +0000 -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback -# -# https://www.linkedin.com/in/harisekhon -# - ---- -name: Fork Update - -on: - workflow_dispatch: - inputs: - debug: - type: string - required: false - schedule: - - cron: '0 10 * * 2' - -jobs: - fork_update: - name: Fork Update - uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update.yaml@master - with: - debug: ${{ github.event.inputs.debug }} - permissions: - contents: write - pull-requests: write From 0215861686be3ee86ae89b6ca2d4de8b087b424f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:49 +0000 Subject: [PATCH 2042/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a92204784..d15d6b66b 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Alpine uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master with: From a7039159e2839a128fcfc223f59c1efadffe54d4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:50 +0000 Subject: [PATCH 2043/2779] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 4b0cb4170..bbcd13ce3 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Alpine 3 uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master with: From 4124f8dc89b2e5d09edbee7a8a7592feaae3d20e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:50 +0000 Subject: [PATCH 2044/2779] updated centos.yaml --- .github/workflows/centos.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index ff1e33598..050cf439d 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: CentOS uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: From 4f207ced80d45d2bfa8c8bbcde54c26d35fa9b86 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:50 +0000 Subject: [PATCH 2045/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 588f5de59..876a9b969 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: CentOS 7 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: From 2a28ec495b2ce83011962023f60efb5fc11edcfd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:51 +0000 Subject: [PATCH 2046/2779] updated centos8.yaml --- .github/workflows/centos8.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 23aeb1c79..a5ec3ec2a 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: CentOS 8 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: From 0ffb0b317883095ab4851936beba920394ac953c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:51 +0000 Subject: [PATCH 2047/2779] updated checkov.yaml --- .github/workflows/checkov.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index a31ba133a..acb4d2f98 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -37,6 +37,7 @@ on: jobs: checkov: + if: github.repository_owner == 'HariSekhon' name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master permissions: From bbf6a1fc07cadd63dff00f7a1674bf8822abd758 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:51 +0000 Subject: [PATCH 2048/2779] updated debian.yaml --- .github/workflows/debian.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 0ac7e1f3e..80950d765 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From 553b673faaf1d44fd1eef2f89dcac8174f572e67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:52 +0000 Subject: [PATCH 2049/2779] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 7763b32f2..bc2d060e1 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 10 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From d53f5159a9951fae92799854f6e4ac7eb5b4071d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:52 +0000 Subject: [PATCH 2050/2779] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 8d043b023..426904fbe 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 8 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From 36d86213770922e4fd401edf45418537893ccff6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 18:51:52 +0000 Subject: [PATCH 2051/2779] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 6a9c34723..490f89be6 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 9 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From fd931e99636e5b1baa003acedb4a9112cf060ebc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:02:18 +0000 Subject: [PATCH 2052/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e71685633..9fa298875 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e71685633a9bf9d8aa19dc9f9597194e4ce5eae4 +Subproject commit 9fa298875fe72586ff61dc2816fb0bc2b0683f93 From 75d5e1f01f342a1199338c3ed8b6eded844ed60a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:02:18 +0000 Subject: [PATCH 2053/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4fa30554f..aaf5fd06b 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4fa30554fc746d5eb90e374c7782bc1bef7f6c47 +Subproject commit aaf5fd06bfa092b87cfbe02557a39678177f8ef6 From a3ebc23158feb58e7210dc97548dd839319122c9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:02:18 +0000 Subject: [PATCH 2054/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 103a1543f..5b548e01b 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 103a1543fa73a0adf4f0af710fcf04d099500170 +Subproject commit 5b548e01bb60dd9a3660e5934f9930f4de10a3b5 From 3b4ea49f097b0dffc89dd0b1cf894cb0c1a5b6a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:02:18 +0000 Subject: [PATCH 2055/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9b14e6b9b..2e4052f28 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9b14e6b9be41fd29ba76a585bd490721be05b2d6 +Subproject commit 2e4052f282510229f388b3a9b589373a11cf1afd From 5449c6c9b5be5626ccf61e8bd326990d878c73e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:15 +0000 Subject: [PATCH 2056/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 99015df89..e00e7a596 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 6 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From 340211a6bf77974c55f5a79b1431be4fc2c73c60 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:15 +0000 Subject: [PATCH 2057/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 3dea635f2..9b43ff3d6 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Debian 7 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: From cc5f606baf71839ae1ce02616aaf5751c002339a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:15 +0000 Subject: [PATCH 2058/2779] updated fedora.yaml --- .github/workflows/fedora.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index c893bc2cf..605ea0075 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Fedora uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master #with: From 73963c1c732510de80c034b76200e707459ee3ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:17 +0000 Subject: [PATCH 2059/2779] updated mac.yaml --- .github/workflows/mac.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index ea6c12828..9a815ad32 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Mac uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: From 10ef68716bc728022dd80874eb1e9fb19675da92 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:17 +0000 Subject: [PATCH 2060/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index b5c50bc4c..89cb7efbc 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Mac 10.15 uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: From 1e0b033015eadc849368c40939e01c6a5ff8e1c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:17 +0000 Subject: [PATCH 2061/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 489d1bcc4..a680ff327 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -27,6 +27,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: PyPy2 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From 47bb0171e97ff9de9d50db4fd03c6276c5665255 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:18 +0000 Subject: [PATCH 2062/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 5c0e12095..451646399 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -27,6 +27,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: PyPy3 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From 72934f8236afea4a8b83cc739c6189865650d1f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:18 +0000 Subject: [PATCH 2063/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index c58d3aaa9..6915ea644 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 2.7 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From 27e1b6e85389944c97f5868bbdd40d9566c36111 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:18 +0000 Subject: [PATCH 2064/2779] updated python3.5.yaml --- .github/workflows/python3.5.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index 659bbc6f6..c0e6338f9 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 3.5 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From bdaec41a4b9235efc09f87211233136a5fbf8eb7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:19 +0000 Subject: [PATCH 2065/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index b325c9523..697177f9d 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -28,10 +28,9 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 3.6 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master - #with: - # debug: 1 with: version: 3.6 #debug: 1 From 091212ddf8c2a0f7e8bdc59e4d8586fd27035a72 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:19 +0000 Subject: [PATCH 2066/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 413e03916..1af8b107b 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 3.7 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From f606e8fcb39eec9ab7e53fce4dda05664640407e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:19 +0000 Subject: [PATCH 2067/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 9621587bb..84178955e 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Python 3.8 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: From 2cd4e2861042109654a16fc27f1130efa89627a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:20 +0000 Subject: [PATCH 2068/2779] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index d18460b45..2de0d75c4 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -35,6 +35,7 @@ on: jobs: semgrep: + if: github.repository_owner == 'HariSekhon' name: Semgrep Cloud uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: From 84a4076e254cf64e3b992e172d9258c010884e94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:20 +0000 Subject: [PATCH 2069/2779] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index af3635d78..692b735f5 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,6 +37,7 @@ on: jobs: semgrep: + if: github.repository_owner == 'HariSekhon' name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master permissions: From 76c380520b2ab92c42cbddeab9bf74aa7cf97cda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:20 +0000 Subject: [PATCH 2070/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 07c4af459..5a803936b 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 63cb1babaf2aacfd5e8c9a40b4dca6c1e1acf6fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:20 +0000 Subject: [PATCH 2071/2779] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 561d7cfda..40f27c66a 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu 14.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 2cec6d28290145168f4e9848cc6673c0c9caa2a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:21 +0000 Subject: [PATCH 2072/2779] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 59d45e0ae..d5a04dc36 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu 16.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 685a4ed5ad26bcbc37e00ea81248067f506f99e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:21 +0000 Subject: [PATCH 2073/2779] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index d3818d27d..da34f3b15 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu 18.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 1e85ac81707ac26e979c576eee257ace838b319d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:21 +0000 Subject: [PATCH 2074/2779] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 7ceee32d7..589268833 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: Ubuntu 20.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: From 5501698916ff88f0b73642037689c1bca20f0729 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:12:22 +0000 Subject: [PATCH 2075/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 4c463496a..c136b4300 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -28,6 +28,7 @@ concurrency: jobs: build: + if: github.repository_owner == 'HariSekhon' name: GitHub Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master #with: From 4cd5d13d059bdc212433041431f127822d55cab9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:15:42 +0000 Subject: [PATCH 2076/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9fa298875..ddc454546 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9fa298875fe72586ff61dc2816fb0bc2b0683f93 +Subproject commit ddc4545469696c0f6e85370312bf95262c720764 From 04c5f7896e45c77f4a05f6af7e5b43f459924ee0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:15:42 +0000 Subject: [PATCH 2077/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index aaf5fd06b..5fa1537ed 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit aaf5fd06bfa092b87cfbe02557a39678177f8ef6 +Subproject commit 5fa1537ed9358b5bba786c80826d3dd71485bd39 From 26be37c87b3019e7cf3b3eed19361f22948de81b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:15:42 +0000 Subject: [PATCH 2078/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 5b548e01b..93f740043 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 5b548e01bb60dd9a3660e5934f9930f4de10a3b5 +Subproject commit 93f740043fc57911b4927172f98c7e74f6d1916f From d5a9d19af5fc728d7176b925e87f943c5c180449 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 17 Feb 2022 19:15:42 +0000 Subject: [PATCH 2079/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2e4052f28..02a6260ff 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2e4052f282510229f388b3a9b589373a11cf1afd +Subproject commit 02a6260ff23b207e9d42506a2b016b9093243018 From b3ec6524b4c0f5469f1e066f07a081e0bde4e75b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 18:51:04 +0000 Subject: [PATCH 2080/2779] switched to more generic disabling repository forks --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_8.yaml | 2 +- .github/workflows/debian_9.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_10.15.yaml | 2 +- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- .github/workflows/python2.7.yaml | 2 +- .github/workflows/python3.5.yaml | 2 +- .github/workflows/python3.6.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index d15d6b66b..0e8afe880 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Alpine uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master with: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index bbcd13ce3..0f9112eb1 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Alpine 3 uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master with: diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 050cf439d..9e3f071e7 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: CentOS uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 876a9b969..275c52047 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: CentOS 7 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index a5ec3ec2a..4fa2d3a34 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: CentOS 8 uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master with: diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index acb4d2f98..c1a53e190 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -37,7 +37,7 @@ on: jobs: checkov: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master permissions: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 80950d765..827ba4458 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index bc2d060e1..c251bb27b 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 10 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 426904fbe..30b677c31 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 8 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 490f89be6..4eb3965b2 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 9 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 605ea0075..1ebb473dc 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Fedora uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master #with: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 9a815ad32..eac5b8d7d 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Mac uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 89cb7efbc..840fc38c3 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Mac 10.15 uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index a680ff327..e3631d1e9 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -27,7 +27,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: PyPy2 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 451646399..a8fba700c 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -27,7 +27,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: PyPy3 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 6915ea644..9a73aedad 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 2.7 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index c0e6338f9..5797d5469 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 3.5 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 697177f9d..d561a026c 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 3.6 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 1af8b107b..2bef6faf8 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 3.7 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 84178955e..c1432111c 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Python 3.8 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 2de0d75c4..9437a66a8 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -35,7 +35,7 @@ on: jobs: semgrep: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Semgrep Cloud uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 692b735f5..216ce7971 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -37,7 +37,7 @@ on: jobs: semgrep: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master permissions: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 5a803936b..9139cdf3a 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 40f27c66a..0ecd25dcd 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu 14.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index d5a04dc36..fca6a638d 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu 16.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index da34f3b15..47f30eb7b 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu 18.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 589268833..5343bf74d 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Ubuntu 20.04 uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master with: diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index c136b4300..6d938de27 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: GitHub Ubuntu uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master #with: From 38146c5d2827314d7b9a1d9e3caf3e2f4e4c9045 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:24:00 +0000 Subject: [PATCH 2081/2779] updated workflows --- .github/workflows/alpine.yaml | 4 ++-- .github/workflows/alpine_3.yaml | 4 ++-- .github/workflows/centos.yaml | 4 ++-- .github/workflows/centos7.yaml | 4 ++-- .github/workflows/centos8.yaml | 4 ++-- .github/workflows/checkov.yaml | 4 ++-- .github/workflows/debian.yaml | 4 ++-- .github/workflows/debian_10.yaml | 4 ++-- .github/workflows/debian_6.yaml.disabled | 6 +++--- .github/workflows/debian_7.yaml.disabled | 6 +++--- .github/workflows/debian_8.yaml | 4 ++-- .github/workflows/debian_9.yaml | 4 ++-- .github/workflows/fedora.yaml | 4 ++-- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/mac.yaml | 4 ++-- .github/workflows/mac_10.15.yaml | 4 ++-- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- .github/workflows/python2.7.yaml | 2 +- .github/workflows/python3.5.yaml | 2 +- .github/workflows/python3.6.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 4 ++-- .github/workflows/semgrep.yaml | 4 ++-- .github/workflows/ubuntu.yaml | 4 ++-- .github/workflows/ubuntu_14.04.yaml | 4 ++-- .github/workflows/ubuntu_16.04.yaml | 4 ++-- .github/workflows/ubuntu_18.04.yaml | 4 ++-- .github/workflows/ubuntu_20.04.yaml | 4 ++-- .github/workflows/ubuntu_github.yaml | 4 ++-- 32 files changed, 57 insertions(+), 57 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 0e8afe880..bf4134967 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 0f9112eb1..c6de94896 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 9e3f071e7..8aae78cc4 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 275c52047..5ae6adf3d 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 4fa2d3a34..3fddfa1c0 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index c1a53e190..12bcbb362 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/templates +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 827ba4458..d022f6947 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index c251bb27b..fbf1f1c8d 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index e00e7a596..1f09cf7b4 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 6 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 9b43ff3d6..2f5e67683 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- @@ -28,7 +28,7 @@ concurrency: jobs: build: - if: github.repository_owner == 'HariSekhon' + if: github.event.repository.fork == false name: Debian 7 uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master with: diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 30b677c31..92410bbb5 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 4eb3965b2..6cb201a1e 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 1ebb473dc..5cae17abe 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index aa17a532e..6d6501c3c 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 66315c238..d3f47e00a 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index eac5b8d7d..83fd21b69 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 840fc38c3..c982de7ec 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index e3631d1e9..dd1e67dd8 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # name: PyPy 2 diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index a8fba700c..3ab09c687 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # name: PyPy 3 diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 9a73aedad..76ddb6c87 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index 5797d5469..cccd0f02e 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index d561a026c..7f6e22b37 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 2bef6faf8..12c884d26 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index c1432111c..64940428c 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 9437a66a8..50d6a29ce 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/templates +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 216ce7971..0de705b2d 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/templates +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 9139cdf3a..ccbf75603 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 0ecd25dcd..a3e08b941 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index fca6a638d..f1ebcd56b 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 47f30eb7b..d412b867f 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 5343bf74d..56b89774a 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 6d938de27..74a224dc3 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -4,11 +4,11 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/DevOps-Python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- From ab151897c3f141533988778a727858353a7c5199 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:27:07 +0000 Subject: [PATCH 2082/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ddc454546..706664f7b 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ddc4545469696c0f6e85370312bf95262c720764 +Subproject commit 706664f7bb88c512bdada2b9480496238896ac35 From 24f6d722c85d5e135445944b78956df9c2ea3378 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:27:07 +0000 Subject: [PATCH 2083/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5fa1537ed..77cf6f982 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5fa1537ed9358b5bba786c80826d3dd71485bd39 +Subproject commit 77cf6f9821c1056c890e845d2044c634f0f6903c From dc49b29e5266a97ef4df5517179d38b646b01171 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:27:07 +0000 Subject: [PATCH 2084/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 93f740043..1774fc022 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 93f740043fc57911b4927172f98c7e74f6d1916f +Subproject commit 1774fc022907673f5c2690bab98dff2e6bbadc50 From c542dcc0745bae6000e4715de53f8e0a88e9a476 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 19:27:07 +0000 Subject: [PATCH 2085/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 02a6260ff..2118def3c 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 02a6260ff23b207e9d42506a2b016b9093243018 +Subproject commit 2118def3c6c352018d9e630753314397bf9a5511 From 8d5cb4da7623b7d905f17f62fab589f1e4a4e97b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:33:14 +0000 Subject: [PATCH 2086/2779] switched to make reusable workflow --- .github/workflows/alpine.yaml | 8 ++++---- .github/workflows/alpine_3.yaml | 8 ++++---- .github/workflows/centos.yaml | 8 ++++---- .github/workflows/centos7.yaml | 8 ++++---- .github/workflows/centos8.yaml | 8 ++++---- .github/workflows/debian.yaml | 8 ++++---- .github/workflows/debian_10.yaml | 8 ++++---- .github/workflows/debian_6.yaml.disabled | 8 ++++---- .github/workflows/debian_7.yaml.disabled | 8 ++++---- .github/workflows/debian_8.yaml | 8 ++++---- .github/workflows/debian_9.yaml | 8 ++++---- .github/workflows/fedora.yaml | 11 ++++++----- .github/workflows/ubuntu.yaml | 8 ++++---- .github/workflows/ubuntu_14.04.yaml | 8 ++++---- .github/workflows/ubuntu_16.04.yaml | 8 ++++---- .github/workflows/ubuntu_18.04.yaml | 8 ++++---- .github/workflows/ubuntu_20.04.yaml | 8 ++++---- .github/workflows/ubuntu_github.yaml | 4 ++-- 18 files changed, 72 insertions(+), 71 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index bf4134967..e1ccb2f59 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Alpine - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + container: alpine:latest #debug: 1 diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index c6de94896..d09654eca 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Alpine 3 - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3 + container: alpine:3 #debug: 1 diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 8aae78cc4..5c7547a49 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + container: centos:latest #debug: 1 diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 5ae6adf3d..8c4dc9246 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 7 + container: centos:7 #debug: 1 diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 3fddfa1c0..910f3728d 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 8 + container: centos:8 #debug: 1 diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d022f6947..e8b5b2cdb 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + container: debian:latest #debug: 1 diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index fbf1f1c8d..523a7487f 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 10 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 10 + container: debian:10 #debug: 1 diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 1f09cf7b4..1192fe4f7 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 6 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 6 + container: debian:6 #debug: 1 diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 2f5e67683..037d7b786 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 7 + container: debian:7 #debug: 1 diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 92410bbb5..514676132 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 8 + container: debian:8 #debug: 1 diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 6cb201a1e..3e2bbcbca 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 9 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 9 + container: debian:9 #debug: 1 diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 5cae17abe..91ff894ce 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -27,9 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Fedora - uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master - #with: - # debug: 1 + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: fedora + #debug: 1 diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index ccbf75603..9aad63ce7 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + container: ubuntu:latest #debug: 1 diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index a3e08b941..b586ad6aa 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 14.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 14.04 + container: ubuntu:14.04 #debug: 1 diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index f1ebcd56b..b021f2b27 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 16.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 16.04 + container: ubuntu:16.04 #debug: 1 diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index d412b867f..c56f3f0af 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 18.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 18.04 + container: ubuntu:18.04 #debug: 1 diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 56b89774a..8812e844c 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 20.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 20.04 + container: ubuntu:20.04 #debug: 1 diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 74a224dc3..7d7d473f3 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -27,9 +27,9 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: GitHub Ubuntu + name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master #with: # debug: 1 From a8c3f391a7a59b8c58a37bda8f8ad093814dc8e6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:42:13 +0000 Subject: [PATCH 2087/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 706664f7b..ce2cb27e3 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 706664f7bb88c512bdada2b9480496238896ac35 +Subproject commit ce2cb27e36aec14f01a89269793a7311d9e3588b From 688101213e83fdc174406851f82a1b5e40c557fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:42:13 +0000 Subject: [PATCH 2088/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 77cf6f982..970d9f9fc 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 77cf6f9821c1056c890e845d2044c634f0f6903c +Subproject commit 970d9f9fc46d4d0c45952f83f7a0c53b5a959269 From 67653e6f7f59de7767a0117bb6dde88720c83be3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:42:13 +0000 Subject: [PATCH 2089/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2118def3c..e0212e355 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2118def3c6c352018d9e630753314397bf9a5511 +Subproject commit e0212e35502da33ecfeb2be88513c9053a6eab78 From aad5ba39ca788874abce90d692a3e0e005b29f2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:48:32 +0000 Subject: [PATCH 2090/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index ce2cb27e3..1db94d597 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit ce2cb27e36aec14f01a89269793a7311d9e3588b +Subproject commit 1db94d597a27f6afe7c74e1d9c83ae3102c94bf7 From 5ff3247784627ea858d93284e7b1240b45d04405 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:48:32 +0000 Subject: [PATCH 2091/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 970d9f9fc..d86ca01bc 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 970d9f9fc46d4d0c45952f83f7a0c53b5a959269 +Subproject commit d86ca01bcfc1f293310bc956c167fdb0c7dc6a2e From 635e14f3f3bd5a6096164fbde553a30d0e9869a7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:48:33 +0000 Subject: [PATCH 2092/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index e0212e355..2d710885e 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit e0212e35502da33ecfeb2be88513c9053a6eab78 +Subproject commit 2d710885e8c010f4de5e852cd91b0be9fc24e3f3 From fa8c87b916392eebe1c72780551025e5d79268cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:54:19 +0000 Subject: [PATCH 2093/2779] added yaml.yaml --- .github/workflows/yaml.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/yaml.yaml diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml new file mode 100644 index 000000000..e939446a6 --- /dev/null +++ b/.github/workflows/yaml.yaml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: YAML + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + check_yaml: + if: github.event.repository.fork == false + name: Check YAML + uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master From 12646de550ba296cf26e4a058e9a1cc1cc723c8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Feb 2022 23:57:45 +0000 Subject: [PATCH 2094/2779] added json.yaml --- .github/workflows/json.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/json.yaml diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml new file mode 100644 index 000000000..d42ced6d8 --- /dev/null +++ b/.github/workflows/json.yaml @@ -0,0 +1,34 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: JSON + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + schedule: + - cron: '0 0 * * 1' + +jobs: + check_json: + if: github.event.repository.fork == false + name: Check JSON + uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master From 6f5a551e7b3328c53ddb3653998c3646fe7b7f3f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 16:27:43 +0000 Subject: [PATCH 2095/2779] added .checkov.yaml --- .checkov.yaml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .checkov.yaml diff --git a/.checkov.yaml b/.checkov.yaml new file mode 100644 index 000000000..8962e2283 --- /dev/null +++ b/.checkov.yaml @@ -0,0 +1,48 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-21 16:53:29 +0000 (Mon, 21 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# C h e c k o v c o n f i g +# ============================================================================ # + +# https://github.com/bridgecrewio/checkov#configuration-using-a-config-file +# +# This is not well documented but the fields seem to be the same as: +# +# checkov --help +# +# See master template at: +# +# https://github.com/HariSekhon/Templates/blob/master/.checkov.yaml + +--- +compact: true +directory: + - . +skip-path: + - bash-tools + - pylib + - sql + - templates +docker-image: harisekhon/pytools +download-external-modules: true # without this gets lots of annoying warning lines such as '2022-02-22 16:14:40,180 [MainThread ] [WARNI] Failed to download module x/y/z:n.n.n' +framework: + - all +no-guide: true +output: cli +quiet: true +repo-id: HariSekhon/DevOps-Python-tools # what to report to Bridgecrew Cloud - without this gets annoying duplicate repos such as 'harisekhon_cli_repo/pytools' +skip-suppressions: true +soft-fail: true From c7467b25b97c47b926929be4d804120d5349b5ce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 18:41:44 +0000 Subject: [PATCH 2096/2779] added .editorconfig --- .editorconfig | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..268166e72 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,78 @@ +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015) +# +# https://github.com/harisekhon/devops-python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# http://www.linkedin.com/in/harisekhon +# + +# http://EditorConfig.org + +# stop recursing upwards for other .editorconfig files +root = true + +# Unix-style newlines with a newline ending every file +[*] +indent_size = 4 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.go] +indent_size = 4 +indent_style = tab +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[Makefile] +indent_size = 4 +indent_style = tab +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[.*] +indent_size = 4 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +# ============================================================================ # +# Older Stuff, don't think I use this any more +# ============================================================================ # + +# Matches multiple files with brace expansion notation +# Set default charset +#[*.{js,py}] +#charset = utf-8 + +# Indentation override for all JS under lib directory +#[lib/**.js] +#indent_style = space +#indent_size = 2 + +# Matches the exact files either package.json or .travis.yml +#[{package.json,.travis.yml}] +#indent_style = space +#indent_size = 2 + +#[*.xml] +#indent_style = space +#indent_size = 2 From bd70bc0208c6ad538db66d2bebfc8a712164ff67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 19:01:49 +0000 Subject: [PATCH 2097/2779] added debug passthrough --- .github/workflows/alpine.yaml | 15 ++++++++++----- .github/workflows/alpine_3.yaml | 15 ++++++++++----- .github/workflows/centos.yaml | 15 ++++++++++----- .github/workflows/centos7.yaml | 15 ++++++++++----- .github/workflows/centos8.yaml | 15 ++++++++++----- .github/workflows/checkov.yaml | 5 +++++ .github/workflows/debian.yaml | 15 ++++++++++----- .github/workflows/debian_10.yaml | 15 ++++++++++----- .github/workflows/debian_6.yaml.disabled | 15 ++++++++++----- .github/workflows/debian_7.yaml.disabled | 15 ++++++++++----- .github/workflows/debian_8.yaml | 15 ++++++++++----- .github/workflows/debian_9.yaml | 15 ++++++++++----- .github/workflows/fedora.yaml | 16 +++++++++++----- .github/workflows/fork-sync.yaml | 3 ++- .github/workflows/fork-update-pr.yaml | 3 ++- .github/workflows/mac.yaml | 7 ++++++- .github/workflows/mac_10.15.yaml | 7 ++++++- .github/workflows/pypy2.yaml | 7 ++++++- .github/workflows/pypy3.yaml | 7 ++++++- .github/workflows/python2.7.yaml | 7 ++++++- .github/workflows/python3.5.yaml | 7 ++++++- .github/workflows/python3.6.yaml | 7 ++++++- .github/workflows/python3.7.yaml | 7 ++++++- .github/workflows/python3.8.yaml | 7 ++++++- .github/workflows/semgrep-cloud.yaml | 5 +++++ .github/workflows/semgrep.yaml | 5 +++++ .github/workflows/ubuntu.yaml | 15 ++++++++++----- .github/workflows/ubuntu_14.04.yaml | 15 ++++++++++----- .github/workflows/ubuntu_16.04.yaml | 15 ++++++++++----- .github/workflows/ubuntu_18.04.yaml | 15 ++++++++++----- .github/workflows/ubuntu_20.04.yaml | 15 ++++++++++----- .github/workflows/ubuntu_github.yaml | 13 +++++++++---- 32 files changed, 253 insertions(+), 100 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index bf4134967..659a61612 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Alpine - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest - #debug: 1 + container: alpine:latest + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index c6de94896..26088e8ec 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Alpine 3 - uses: HariSekhon/GitHub-Actions/.github/workflows/alpine.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3 - #debug: 1 + container: alpine:3 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 8aae78cc4..901a90e9f 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest - #debug: 1 + container: centos:latest + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 5ae6adf3d..6e3ebb0e6 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 7 - #debug: 1 + container: centos:7 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 3fddfa1c0..e782ababf 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: CentOS 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/centos.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 8 - #debug: 1 + container: centos:8 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 12bcbb362..85e629364 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -32,6 +32,11 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d022f6947..ed706c124 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest - #debug: 1 + container: debian:latest + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index fbf1f1c8d..e2aa2395d 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 10 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 10 - #debug: 1 + container: debian:10 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 1f09cf7b4..c04220d48 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 6 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 6 - #debug: 1 + container: debian:6 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 2f5e67683..26f23cce1 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 7 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 7 - #debug: 1 + container: debian:7 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 92410bbb5..d551a85c0 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 8 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 8 - #debug: 1 + container: debian:8 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 6cb201a1e..d92ec4607 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Debian 9 - uses: HariSekhon/GitHub-Actions/.github/workflows/debian.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 9 - #debug: 1 + container: debian:9 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 5cae17abe..a83cad0c7 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,9 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Fedora - uses: HariSekhon/GitHub-Actions/.github/workflows/fedora.yaml@master - #with: - # debug: 1 + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: fedora + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 6d6501c3c..98331b7d8 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -18,8 +18,9 @@ on: workflow_dispatch: inputs: debug: - type: string + type: boolean required: false + default: false schedule: - cron: '0 * * * *' diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index d3f47e00a..d3b156e38 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -18,8 +18,9 @@ on: workflow_dispatch: inputs: debug: - type: string + type: boolean required: false + default: false schedule: # fork-sync happens for default branch every hour, so just after that, run PRs for branches - cron: '2 10 * * 2' diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 83fd21b69..09049616c 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: version: latest - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index c982de7ec..8f328ca80 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: version: 10.15 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index dd1e67dd8..a625697b6 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -18,6 +18,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -32,4 +37,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy2 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 3ab09c687..c0ba3825e 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -18,6 +18,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -32,4 +37,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy3 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 76ddb6c87..beca7bcda 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 2.7 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index cccd0f02e..fa0f7ddbf 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.5 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 7f6e22b37..5639271ab 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.6 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 12c884d26..25a455a85 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.7 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 64940428c..095e991ac 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -33,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.8 - #debug: 1 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 50d6a29ce..0b7e88f01 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -30,6 +30,11 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 0de705b2d..ecb59b85f 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -32,6 +32,11 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index ccbf75603..ff42db345 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest - #debug: 1 + container: ubuntu:latest + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index a3e08b941..53a46cf08 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 14.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 14.04 - #debug: 1 + container: ubuntu:14.04 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index f1ebcd56b..44c7399e7 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 16.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 16.04 - #debug: 1 + container: ubuntu:16.04 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index d412b867f..e8aca96c3 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 18.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 18.04 - #debug: 1 + container: ubuntu:18.04 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 56b89774a..f5e017c96 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,10 +32,10 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Ubuntu 20.04 - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 20.04 - #debug: 1 + container: ubuntu:20.04 + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 74a224dc3..f6a2a69f4 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -19,6 +19,11 @@ on: branches: - master workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 7 * * *' @@ -27,9 +32,9 @@ concurrency: cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: GitHub Ubuntu + name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master - #with: - # debug: 1 + with: + # debug: true From f8ba2a996acadf6c110c383806f218a1f73d8851 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 19:53:56 +0000 Subject: [PATCH 2098/2779] moved permissions key to top and added missing concurrency build auto-cancellations --- .github/workflows/checkov.yaml | 11 +++++++---- .github/workflows/fork-sync.yaml | 9 +++++++-- .github/workflows/fork-update-pr.yaml | 11 ++++++++--- .github/workflows/semgrep-cloud.yaml | 4 ++++ .github/workflows/semgrep.yaml | 13 +++++++++---- .github/workflows/ubuntu_github.yaml | 2 +- 6 files changed, 36 insertions(+), 14 deletions(-) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 85e629364..d05b53148 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -40,12 +40,15 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + actions: read + contents: read + security-events: write + jobs: checkov: if: github.event.repository.fork == false name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master - permissions: - actions: read - contents: read - security-events: write + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 98331b7d8..fd91109e2 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -24,6 +24,13 @@ on: schedule: - cron: '0 * * * *' +permissions: + contents: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false + jobs: fork_sync: if: github.repository_owner != 'HariSekhon' @@ -31,5 +38,3 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master with: debug: ${{ github.event.inputs.debug }} - permissions: - contents: write diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index d3b156e38..12449fb36 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -25,6 +25,14 @@ on: # fork-sync happens for default branch every hour, so just after that, run PRs for branches - cron: '2 10 * * 2' +permissions: + contents: write + pull-requests: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: false + jobs: fork_update_pr: if: github.repository_owner != 'HariSekhon' @@ -32,6 +40,3 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master with: debug: ${{ github.event.inputs.debug }} - permissions: - contents: write - pull-requests: write diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 0b7e88f01..da4ccbdbd 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -38,6 +38,10 @@ on: schedule: - cron: '0 0 * * 1' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: semgrep: if: github.event.repository.fork == false diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index ecb59b85f..410e63c0f 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -40,12 +40,17 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: semgrep: if: github.event.repository.fork == false name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master - permissions: - actions: read - contents: read - security-events: write diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index f6a2a69f4..83a9e46c8 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -37,4 +37,4 @@ jobs: name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master with: - # debug: true + debug: ${{ github.event.inputs.debug }} From bbf552ad3fbf71333816f360c1c55daaaada8640 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 20:06:59 +0000 Subject: [PATCH 2099/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1db94d597..57ea4e7cf 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1db94d597a27f6afe7c74e1d9c83ae3102c94bf7 +Subproject commit 57ea4e7cf483d6faecf630bc0b1173f80ab127d4 From acc023e945f335442b2256472144cc9983e05802 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 20:06:59 +0000 Subject: [PATCH 2100/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d86ca01bc..eae777b99 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d86ca01bcfc1f293310bc956c167fdb0c7dc6a2e +Subproject commit eae777b99fb25d19587b0c0f3d22c1f838ecf169 From 132b991b108146fd9813f212229aa409e3c86c08 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 20:06:59 +0000 Subject: [PATCH 2101/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 1774fc022..dc6c1cd42 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 1774fc022907673f5c2690bab98dff2e6bbadc50 +Subproject commit dc6c1cd4278bda75c73dad895daa72846f4ff8e9 From b118ae872145afa0b07ad8dd5814ab11ca80e145 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 22 Feb 2022 20:06:59 +0000 Subject: [PATCH 2102/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2d710885e..1afa44055 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2d710885e8c010f4de5e852cd91b0be9fc24e3f3 +Subproject commit 1afa44055e21a2ec8631f298e13898e7f1c8bdc3 From 72b56ece611667531b519adcf600f52a97395912 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Feb 2022 00:01:01 +0000 Subject: [PATCH 2103/2779] tweaked debug pass through as github.event.inputs.debug isn't present in scheduled runs --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml | 2 +- .github/workflows/debian_9.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/json.yaml | 5 +++++ .github/workflows/mac.yaml | 2 +- .github/workflows/mac_10.15.yaml | 2 +- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- .github/workflows/python2.7.yaml | 2 +- .github/workflows/python3.5.yaml | 2 +- .github/workflows/python3.6.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/yaml.yaml | 9 +++++++++ 32 files changed, 44 insertions(+), 30 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 659a61612..21aacec90 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 26088e8ec..6aaab2a03 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:3 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 901a90e9f..1c611d673 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 6e3ebb0e6..4454ab3d9 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:7 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index e782ababf..4ba82161d 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:8 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index d05b53148..2e57716ad 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -51,4 +51,4 @@ jobs: name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master with: - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index ed706c124..a733a4647 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index e2aa2395d..813fbd9f9 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:10 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index c04220d48..94a1d5176 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:6 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 26f23cce1..5ccf6bb63 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:7 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index d551a85c0..613e1e15e 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:8 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index d92ec4607..398709d88 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:9 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index a83cad0c7..49924b3fb 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: fedora - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index fd91109e2..187feb41e 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -37,4 +37,4 @@ jobs: name: Fork Sync uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master with: - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 12449fb36..e4d89e440 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -39,4 +39,4 @@ jobs: name: Fork Update PR uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master with: - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index d42ced6d8..4dab279cc 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -24,6 +24,11 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 09049616c..08766ad90 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: version: latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 8f328ca80..4d0316349 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master with: version: 10.15 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index a625697b6..cb0cfc3af 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -37,4 +37,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy2 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index c0ba3825e..fdfc2d794 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -37,4 +37,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy3 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index beca7bcda..e1d3fced4 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 2.7 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml index fa0f7ddbf..5ebb24d18 100644 --- a/.github/workflows/python3.5.yaml +++ b/.github/workflows/python3.5.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.5 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 5639271ab..60a3b9840 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.6 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 25a455a85..093d7d27b 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.7 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 095e991ac..c925e6db7 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.8 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index ff42db345..9deae95db 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:latest - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 53a46cf08..be93c352b 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:14.04 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 44c7399e7..d5c3a1924 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:16.04 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index e8aca96c3..b363c2a92 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:18.04 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index f5e017c96..f0d393ffc 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -38,4 +38,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:20.04 - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 83a9e46c8..0f6866651 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -37,4 +37,4 @@ jobs: name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master with: - debug: ${{ github.event.inputs.debug }} + debug: ${{ github.event.inputs.debug || false }} diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index e939446a6..d416105c6 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -24,9 +24,18 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: check_yaml: if: github.event.repository.fork == false From be079535eb00b469161bc52bf3345730edd31d63 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 23 Feb 2022 23:37:09 +0000 Subject: [PATCH 2104/2779] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 187feb41e..f139e9617 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -22,7 +22,7 @@ on: required: false default: false schedule: - - cron: '0 * * * *' + - cron: '0 9 * * *' permissions: contents: write From 2953d2da34054e172c99dd37bd5d8aad2d7217e2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 11:35:07 +0000 Subject: [PATCH 2105/2779] updated validate.yaml --- .github/workflows/validate.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index cf901ed71..0594bda8b 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- @@ -24,10 +24,20 @@ on: - master - main workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false schedule: - cron: '0 0 * * 1' +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + jobs: validate: + if: github.event.repository.fork == false name: Validate uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master From bcbfaa4460c4edf6073325a9203624620d8281ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 14:00:01 +0000 Subject: [PATCH 2106/2779] updated CODEOWNERS --- .github/CODEOWNERS | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 636853854..962ae64c7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,4 +17,10 @@ # Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam -#* @harisekhon +# * includes changes under .github/ + +#* @harisekhon +#* @myorg/platform-engineering # team based is the way to go +#* @myorg/devops +#k8s @myorg/devops +#src/* @myorg/developers From 198336c212933ea4d17dad22b54ab9927cf6a586 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 14:50:22 +0000 Subject: [PATCH 2107/2779] updated CODEOWNERS --- .github/CODEOWNERS | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 962ae64c7..ad861088b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,10 +17,22 @@ # Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam +# Tips: +# # * includes changes under .github/ +# dir/* only matches first level file changes but doesn't recurse +# dir/ recurses +# +# - CODEOWNERS in base branch of PR determines review request +# - paths are case sensitive +# - last match wins, use * at top for overall owner then override with more specific teams -#* @harisekhon -#* @myorg/platform-engineering # team based is the way to go -#* @myorg/devops -#k8s @myorg/devops -#src/* @myorg/developers +#* @harisekhon # username or email address +#* @myorg/platform-engineering # team based is the way to go - team must have Write access to the repo regardless of if individuals have access +#* @myorg/devops +#k8s @myorg/devops @myorg/sre-team +#apps/ @myorg/developers +#apps/dir2 # ignores dir2 as no owner/team specified on this line +#src/ @myorg/developers +#docs/ docs@example.com +#.github/workflows @ci-cd-team From 57d47f1ed5133cef5d9f04abdfa22fcd34404e99 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:29:47 +0000 Subject: [PATCH 2108/2779] added codeowners.yaml --- .github/workflows/codeowners.yaml | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/codeowners.yaml diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml new file mode 100644 index 000000000..c0ae12715 --- /dev/null +++ b/.github/workflows/codeowners.yaml @@ -0,0 +1,43 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Codeowners + +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + - main + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + validate: + if: github.event.repository.fork == false + name: Validate CODEOWNERS + uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master From ed2d55fb11c9fc94dd622d3958ce957c50af1360 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:56:55 +0000 Subject: [PATCH 2109/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 57ea4e7cf..fcd7360af 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 57ea4e7cf483d6faecf630bc0b1173f80ab127d4 +Subproject commit fcd7360af7320590dcae536dafabfbfa3afae11e From 899248b936162e84d0fa5035838830bf42ce07d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:56:55 +0000 Subject: [PATCH 2110/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index eae777b99..6e7b0b552 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit eae777b99fb25d19587b0c0f3d22c1f838ecf169 +Subproject commit 6e7b0b552fd5ef9d0ad885f9909c6d8e5b765d2f From 14dc0fe8aabb0353b9f4c12436adc0b145b50df9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:56:55 +0000 Subject: [PATCH 2111/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index dc6c1cd42..0ca10f23b 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit dc6c1cd4278bda75c73dad895daa72846f4ff8e9 +Subproject commit 0ca10f23bde76e0533c5dc85cfc325dd641de9ee From 8339dc2c903550a25907181ea8cbc1344a90398d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Feb 2022 20:56:55 +0000 Subject: [PATCH 2112/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 1afa44055..c45cb02b5 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 1afa44055e21a2ec8631f298e13898e7f1c8bdc3 +Subproject commit c45cb02b56adec293d593ea501cf51821a24c090 From dacbcedd624f659676edbccb952c539e636adf51 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Feb 2022 10:45:30 +0000 Subject: [PATCH 2113/2779] renamed .github/workflows/python3.5.yaml to .github/workflows/python3.10.yaml --- .github/workflows/{python3.5.yaml => python3.10.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python3.5.yaml => python3.10.yaml} (100%) diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.10.yaml similarity index 100% rename from .github/workflows/python3.5.yaml rename to .github/workflows/python3.10.yaml From 028eca92ea47ec8f2ebdccba6c7ff160c53497da Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Feb 2022 10:45:54 +0000 Subject: [PATCH 2114/2779] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 5ebb24d18..59657673a 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -12,7 +12,7 @@ # --- -name: Python 3.5 +name: Python 3.10 on: push: @@ -34,8 +34,8 @@ concurrency: jobs: build: if: github.event.repository.fork == false - name: Python 3.5 + name: Python 3.10 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: - version: 3.5 + version: 3.10 debug: ${{ github.event.inputs.debug || false }} From b7960babb4b69af2181d1f24410c12470f7e028c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Feb 2022 10:47:21 +0000 Subject: [PATCH 2115/2779] added python3.9.yaml --- .github/workflows/python3.9.yaml | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/python3.9.yaml diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml new file mode 100644 index 000000000..f1ae3982c --- /dev/null +++ b/.github/workflows/python3.9.yaml @@ -0,0 +1,41 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.9 + +on: + push: + branches: + - master + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + if: github.event.repository.fork == false + name: Python 3.9 + uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + with: + version: 3.9 + debug: ${{ github.event.inputs.debug || false }} From afc0fdf8f8d6d3b00e9ea5bc8ceefe8057869abf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Feb 2022 10:48:10 +0000 Subject: [PATCH 2116/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2445c90db..ae1ff5cdb 100644 --- a/README.md +++ b/README.md @@ -91,10 +91,11 @@ Hari Sekhon - DevOps Python Tools [![Python versions](https://img.shields.io/badge/Python-2.7+-3776AB?logo=python&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools) [![Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+2.7%22) -[![Python 3.5](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.5/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.5%22) [![Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.6%22) [![Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22) [![Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22) +[![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) +[![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) [![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) [![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) From 9d797c6972f3379773e77c9d00f2b6076c9927c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 13:28:18 +0000 Subject: [PATCH 2117/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 0f6866651..906236af4 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -35,6 +35,6 @@ jobs: make: if: github.event.repository.fork == false name: Make - uses: HariSekhon/GitHub-Actions/.github/workflows/ubuntu_github.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: debug: ${{ github.event.inputs.debug || false }} From e633e2a6e5f2485886d698b402b801d6b6b8270f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:02 +0000 Subject: [PATCH 2118/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 21aacec90..ed007352a 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:latest + caches: apk pip cpanm debug: ${{ github.event.inputs.debug || false }} From 5b2b31359a3e1a05e9ba20a67f3cf5a956d47ded Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 2119/2779] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 6aaab2a03..45290c0db 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:3 + caches: apk pip cpanm debug: ${{ github.event.inputs.debug || false }} From 04e7be1742349530646bed0ad8b494a17b3ba697 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 2120/2779] updated centos.yaml --- .github/workflows/centos.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 1c611d673..9e3306bfc 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:latest + caches: yum pip cpanm debug: ${{ github.event.inputs.debug || false }} From 1e4ea79fc79a5a0b34274828c1175909ff468636 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 2121/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 4454ab3d9..7f116373b 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:7 + caches: yum pip cpanm debug: ${{ github.event.inputs.debug || false }} From 39fb995f35275e7c7bb5597bc3e078f302fb61b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 2122/2779] updated centos8.yaml --- .github/workflows/centos8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 4ba82161d..20dfc35c4 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:8 + caches: yum pip cpanm debug: ${{ github.event.inputs.debug || false }} From a3638521c5c848280b50141cacb50704297ec094 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 2123/2779] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index c0ae12715..d4e575f30 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -32,6 +32,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 15c0be92da5ab97a5002f833821ae79611c77ae4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:03 +0000 Subject: [PATCH 2124/2779] updated debian.yaml --- .github/workflows/debian.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index a733a4647..c3fd4fc9b 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:latest + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From fb319ef5d2c89609c9a55a3f5f159573a7013877 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 2125/2779] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 813fbd9f9..b76dcd103 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:10 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From c6655dae1c7a791a247f70c93f3e82e6c7a63807 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 2126/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 94a1d5176..b11a56258 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:6 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From e82dfd8d2ed275527c78a0e0a00a3168dfcdfa81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 2127/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 5ccf6bb63..e523af78f 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:7 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 364a8dded1efecb9b39d0c75fe597788c07c580a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 2128/2779] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 613e1e15e..fe764801c 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:8 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 1dd4ddaf2f51b546e784a5ea057e24b7e629f762 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:04 +0000 Subject: [PATCH 2129/2779] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 398709d88..f473395da 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:9 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 7abd77f2fed7bbaff7c7502c5de8ba3efa2ed796 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 2130/2779] updated fedora.yaml --- .github/workflows/fedora.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 49924b3fb..b29f4c229 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: fedora + caches: yum pip cpanm debug: ${{ github.event.inputs.debug || false }} From 05a4812c47c89ecfda6f4ce3c0600e8cbe0e0975 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 2131/2779] updated json.yaml --- .github/workflows/json.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index 4dab279cc..b66e8cee6 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -32,6 +32,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + jobs: check_json: if: github.event.repository.fork == false From 985f82224ea0755d8343c91bce5c3b2956adf4e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 2132/2779] updated mac.yaml --- .github/workflows/mac.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 08766ad90..bbee846e8 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -27,15 +27,19 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Mac - uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: latest + runs-on: macos-latest + caches: brew pip cpanm debug: ${{ github.event.inputs.debug || false }} From 5be50a20225e0ffc971588c2564668c55461fa3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 2133/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index 4d0316349..f2f6a57ac 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -27,15 +27,19 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true jobs: - build: + make: if: github.event.repository.fork == false - name: Mac 10.15 - uses: HariSekhon/GitHub-Actions/.github/workflows/mac.yaml@master + name: Make + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 10.15 + runs-on: macos-10.15 + caches: brew pip cpanm debug: ${{ github.event.inputs.debug || false }} From 56a1eb7cc5e4b4fc4037c4853ddbe96e0e70bbdc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:05 +0000 Subject: [PATCH 2134/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index cb0cfc3af..c89f841fa 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -26,6 +26,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 21c36b043001da8af2014039880750cab988ef62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 2135/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index fdfc2d794..ecdf83762 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -26,6 +26,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 6361ac5869bed7de5c42a6f20191db8443aadcfd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 2136/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index e1d3fced4..53b6b6171 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From accbbef60a0336d5c6bd7ec017d639fce60add68 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 2137/2779] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 59657673a..3552a5a98 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -37,5 +40,5 @@ jobs: name: Python 3.10 uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: - version: 3.10 + version: "3.10" debug: ${{ github.event.inputs.debug || false }} From cea4bb065d3c595153700cc7af6c1259cc979aef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 2138/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 60a3b9840..b3e23f7fa 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From b985dce49d26e14b539918c4e30692e6c7730319 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 2139/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 093d7d27b..168caa2e8 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From fc716d57a8a07692fe2952cd5f3e295a6e7e8b7f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:06 +0000 Subject: [PATCH 2140/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index c925e6db7..b8ae52619 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From d579d1734efff4a3b0ac8d6a9dc8b822deff5d7d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 2141/2779] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index f1ae3982c..580830e31 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 9f83f0bbdd25288c15c806a4a50183fab050692b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 2142/2779] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index da4ccbdbd..1e8278f7e 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -38,6 +38,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From b4ec7c3d6393a8e87af2d87e4d56c1186e6c78d1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 2143/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 9deae95db..20c11bc5e 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:latest + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From ac808c2b717941eccdfbdc89645106856bde240b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 2144/2779] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index be93c352b..433a138f5 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:14.04 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From f7651ec3b60ceb6d508e6ac157e30f7dbe87a243 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:07 +0000 Subject: [PATCH 2145/2779] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index d5c3a1924..46daed829 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:16.04 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 1c113092e3aa8979b36d043c2383e4914293238f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 2146/2779] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index b363c2a92..b22c72cd9 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:18.04 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From de492e31675c57a903257018e443f6abf07fa8d2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 2147/2779] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index f0d393ffc..cc27c8016 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -38,4 +41,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:20.04 + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From f89e3967dc0e99ee20748de1ae4e3c2b73bbf920 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 2148/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 906236af4..323631dca 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -27,6 +27,9 @@ on: schedule: - cron: '0 7 * * *' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true @@ -37,4 +40,5 @@ jobs: name: Make uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: + caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 5b9740262c95f478d89fd83655326f18fa4b69d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 2149/2779] updated validate.yaml --- .github/workflows/validate.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 0594bda8b..bbdc02826 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -32,6 +32,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 376e27aee3256e04d38d325a10a20735122dd9f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:39:08 +0000 Subject: [PATCH 2150/2779] updated yaml.yaml --- .github/workflows/yaml.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index d416105c6..c11764e11 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -32,6 +32,9 @@ on: schedule: - cron: '0 0 * * 1' +permissions: + contents: read + concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true From 09a740dc794c03556ab0503bf60e8df6b25c7e59 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:47:56 +0000 Subject: [PATCH 2151/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index b11a56258..bb6698b9a 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -41,5 +41,6 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:6 - caches: apt pip cpanm + # causes nodejs errors + #caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From b42d05f595a680ed4a097a3ef883bcf54e2b27f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:47:57 +0000 Subject: [PATCH 2152/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index e523af78f..216adb436 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -41,5 +41,6 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:7 - caches: apt pip cpanm + # causes nodejs errors + #caches: apt pip cpanm debug: ${{ github.event.inputs.debug || false }} From 850f12f5d9a678c6438009a78d54fa68eddd0606 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:52:40 +0000 Subject: [PATCH 2153/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fcd7360af..1f9ee12e5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fcd7360af7320590dcae536dafabfbfa3afae11e +Subproject commit 1f9ee12e52063c59c0620842bfe7186e91bec5a8 From e7c80a8460b1d227ac466d3a41da480505e5f374 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:52:40 +0000 Subject: [PATCH 2154/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6e7b0b552..aec17428f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6e7b0b552fd5ef9d0ad885f9909c6d8e5b765d2f +Subproject commit aec17428f1a946d4b2ead6b37379b84c9b625a33 From b16b3c9c5d249f98c60d01dab8e777443421f93b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:52:40 +0000 Subject: [PATCH 2155/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 0ca10f23b..69aff25cb 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 0ca10f23bde76e0533c5dc85cfc325dd641de9ee +Subproject commit 69aff25cbac90c8fc97a66c0d9e4351ea659c54e From 06840d5c99cc0a788d18705e5498e118dd112712 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Mar 2022 19:52:40 +0000 Subject: [PATCH 2156/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c45cb02b5..e444f83ec 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c45cb02b56adec293d593ea501cf51821a24c090 +Subproject commit e444f83ecb0082c0639c4df23ca24e7790955d7b From 34e75fb374570061730abff675e28baac361566c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 20 Mar 2022 19:04:09 +0000 Subject: [PATCH 2157/2779] updated .pylintrc --- .pylintrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 65e97ab22..cd3e053f6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -59,7 +59,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111 +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111,consider-using-f-string [REPORTS] @@ -108,7 +108,7 @@ name-group= include-naming-hint=no # Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ +function-rgx=[A-Za-z_][A-Za-z0-9_]{2,30}$ # Naming hint for function names function-name-hint=[a-z_][a-z0-9_]{2,30}$ From 629beaff5a425be174916121208bc981d50ceb3c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 20 Mar 2022 19:08:08 +0000 Subject: [PATCH 2158/2779] updated .pylintrc --- .pylintrc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index cd3e053f6..c20ac8f94 100644 --- a/.pylintrc +++ b/.pylintrc @@ -274,7 +274,15 @@ ignored-modules= # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). This supports can work # with qualified names. -ignored-classes= +# +# avoids the following error: +# +# pylint -E ./check_zookeeper_version.py +# +# E: 69,12: Instance of '_socketobject' has no 'sendall' member (no-member) +# E: 70,19: Instance of '_socketobject' has no 'recv' member (no-member) +# +ignored-classes=SQLObject,_socketobject # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular From 0a61737c56e004ea6cb0a3a2293b9a5ba7001919 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Mar 2022 18:47:59 +0000 Subject: [PATCH 2159/2779] updated debug passthrough --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml | 2 +- .github/workflows/centos7.yaml | 2 +- .github/workflows/centos8.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml | 2 +- .github/workflows/debian_9.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_10.15.yaml | 2 +- .github/workflows/pypy2.yaml | 2 +- .github/workflows/pypy3.yaml | 2 +- .github/workflows/python2.7.yaml | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.6.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml | 2 +- .github/workflows/ubuntu_16.04.yaml | 2 +- .github/workflows/ubuntu_18.04.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- 31 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index ed007352a..287b2d490 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -42,4 +42,4 @@ jobs: with: container: alpine:latest caches: apk pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 45290c0db..96f31e3ab 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -42,4 +42,4 @@ jobs: with: container: alpine:3 caches: apk pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index 9e3306bfc..ed6372ec7 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -42,4 +42,4 @@ jobs: with: container: centos:latest caches: yum pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 7f116373b..681be974f 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -42,4 +42,4 @@ jobs: with: container: centos:7 caches: yum pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 20dfc35c4..9c4311285 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -42,4 +42,4 @@ jobs: with: container: centos:8 caches: yum pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 2e57716ad..d05b53148 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -51,4 +51,4 @@ jobs: name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master with: - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index c3fd4fc9b..ae2afe46e 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -42,4 +42,4 @@ jobs: with: container: debian:latest caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index b76dcd103..2656f55fd 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -42,4 +42,4 @@ jobs: with: container: debian:10 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index bb6698b9a..dff981bd2 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -43,4 +43,4 @@ jobs: container: debian:6 # causes nodejs errors #caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 216adb436..74b06b44e 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -43,4 +43,4 @@ jobs: container: debian:7 # causes nodejs errors #caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index fe764801c..6b2a6f979 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -42,4 +42,4 @@ jobs: with: container: debian:8 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index f473395da..3641c892c 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -42,4 +42,4 @@ jobs: with: container: debian:9 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index b29f4c229..2128939b8 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -42,4 +42,4 @@ jobs: with: container: fedora caches: yum pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index f139e9617..556bd80d9 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -37,4 +37,4 @@ jobs: name: Fork Sync uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master with: - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index e4d89e440..12449fb36 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -39,4 +39,4 @@ jobs: name: Fork Update PR uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master with: - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index bbee846e8..81aaec5b1 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -42,4 +42,4 @@ jobs: with: runs-on: macos-latest caches: brew pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index f2f6a57ac..db4781fab 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -42,4 +42,4 @@ jobs: with: runs-on: macos-10.15 caches: brew pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index c89f841fa..ae3774daf 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -40,4 +40,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy2 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index ecdf83762..966741006 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -40,4 +40,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: pypy3 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 53b6b6171..bb03008d4 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 2.7 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 3552a5a98..4ba2d2cdf 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: "3.10" - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index b3e23f7fa..d94daf551 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.6 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 168caa2e8..9e2167f91 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.7 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index b8ae52619..0adb02bb5 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.8 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 580830e31..04a325e29 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master with: version: 3.9 - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 20c11bc5e..927f7edc2 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:latest caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 433a138f5..fff40eaa6 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:14.04 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 46daed829..cd96b0858 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:16.04 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index b22c72cd9..385bcfff6 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:18.04 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index cc27c8016..360a8db4e 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -42,4 +42,4 @@ jobs: with: container: ubuntu:20.04 caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 323631dca..59a9c9110 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -41,4 +41,4 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: caches: apt pip cpanm - debug: ${{ github.event.inputs.debug || false }} + debug: ${{ github.event.inputs.debug }} From 11a554febd4d3764100e4494217e214e2c5a93c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 12:26:46 +0100 Subject: [PATCH 2160/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae1ff5cdb..b0e453c94 100644 --- a/README.md +++ b/README.md @@ -47,13 +47,13 @@ Hari Sekhon - DevOps Python Tools [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) -[![Shippable](https://img.shields.io/shippable/5e52c63445c70f0007ff5144/master?label=Shippable&logo=jfrog)](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) +[![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml) [![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) From 5b51de257400d01702714b56e5786d2569ca3f55 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 12:31:39 +0100 Subject: [PATCH 2161/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b0e453c94..ec5ba95b9 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,12 @@ Hari Sekhon - DevOps Python Tools [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) +[![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) +[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) -[![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) [![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) -[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) From 9e32b2e011867d59a73cb650fe887c95fdc4ac7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 2162/2779] renamed dockerhub_pytools_alpine.yaml to docker_pytools_alpine.yaml --- .../{dockerhub_pytools_alpine.yaml => docker_pytools_alpine.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_alpine.yaml => docker_pytools_alpine.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_alpine.yaml rename to .github/workflows/docker_pytools_alpine.yaml From 5155a7898353142de3a40954d1323315d8d5040e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 2163/2779] renamed dockerhub_pytools_centos.yaml to docker_pytools_centos.yaml --- .../{dockerhub_pytools_centos.yaml => docker_pytools_centos.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_centos.yaml => docker_pytools_centos.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_centos.yaml rename to .github/workflows/docker_pytools_centos.yaml From 90d8b86bc867116df6489163321cfe504f2062fa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 2164/2779] renamed dockerhub_pytools_debian.yaml to docker_pytools_debian.yaml --- .../{dockerhub_pytools_debian.yaml => docker_pytools_debian.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_debian.yaml => docker_pytools_debian.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_debian.yaml rename to .github/workflows/docker_pytools_debian.yaml From 33a93df4781346aedf83f7ae33fe5cf9ef8f50fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 2165/2779] renamed dockerhub_pytools_fedora.yaml to docker_pytools_fedora.yaml --- .../{dockerhub_pytools_fedora.yaml => docker_pytools_fedora.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_fedora.yaml => docker_pytools_fedora.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_fedora.yaml rename to .github/workflows/docker_pytools_fedora.yaml From 0f8f7a486e261d442b3aa8d1af7338895b77085f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:14:50 +0100 Subject: [PATCH 2166/2779] renamed dockerhub_pytools_ubuntu.yaml to docker_pytools_ubuntu.yaml --- .../{dockerhub_pytools_ubuntu.yaml => docker_pytools_ubuntu.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{dockerhub_pytools_ubuntu.yaml => docker_pytools_ubuntu.yaml} (100%) diff --git a/.github/workflows/dockerhub_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml similarity index 100% rename from .github/workflows/dockerhub_pytools_ubuntu.yaml rename to .github/workflows/docker_pytools_ubuntu.yaml From 338a6c8bc3886c233dcbb9ea394fc4349c8f8c2d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:25 +0100 Subject: [PATCH 2167/2779] updated docker_pytools_alpine.yaml --- .github/workflows/docker_pytools_alpine.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index ff2197707..c3687b351 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (Alpine) +name: Docker Build (Alpine) on: push: @@ -26,8 +26,9 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: alpine + repo_tags: | + harisekhon/pytools:alpine + ghcr.io/harisekhon/pytools:alpine dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-alpine secrets: From fe9bed2863a05acd3df3b931a193faa55afb17bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:26 +0100 Subject: [PATCH 2168/2779] updated docker_pytools_centos.yaml --- .github/workflows/docker_pytools_centos.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml index 06778bd22..92b36e7d5 100644 --- a/.github/workflows/docker_pytools_centos.yaml +++ b/.github/workflows/docker_pytools_centos.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (CentOS) +name: Docker Build (CentOS) on: push: @@ -26,8 +26,9 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: latest centos + repo_tags: | + harisekhon/pytools:centos + ghcr.io/harisekhon/pytools:centos dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-centos secrets: From cdfa7e53803c7558f64d9e8932d4a1384120640e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:29 +0100 Subject: [PATCH 2169/2779] updated docker_pytools_debian.yaml --- .github/workflows/docker_pytools_debian.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index aa9da7e6a..8b34df315 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (Debian) +name: Docker Build (Debian) on: push: @@ -26,8 +26,9 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: debian + repo_tags: | + harisekhon/pytools:debian + ghcr.io/harisekhon/pytools:debian dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-debian secrets: From e808440babee55333beea4a93dd027c71ed585e5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:31 +0100 Subject: [PATCH 2170/2779] updated docker_pytools_fedora.yaml --- .github/workflows/docker_pytools_fedora.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 3aa3c800f..190cf703b 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (Fedora) +name: Docker Build (Fedora) on: push: @@ -26,8 +26,9 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: fedora + repo_tags: | + harisekhon/pytools:fedora + ghcr.io/harisekhon/pytools:fedora dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-fedora secrets: From 3deb0857ff51081a3eda899d25a93975e095a213 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:16:32 +0100 Subject: [PATCH 2171/2779] updated docker_pytools_ubuntu.yaml --- .github/workflows/docker_pytools_ubuntu.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index d53930274..fe5e51a95 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -12,7 +12,7 @@ # --- -name: DockerHub Build (Ubuntu) +name: Docker Build (Ubuntu) on: push: @@ -26,8 +26,11 @@ jobs: name: Docker Build uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master with: - repo: harisekhon/pytools - tags: ubuntu latest + repo_tags: | + harisekhon/pytools:latest + harisekhon/pytools:ubuntu + ghcr.io/harisekhon/pytools:latest + ghcr.io/harisekhon/pytools:ubuntu dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-ubuntu secrets: From da4664c347c765612a4ca40c698f18c191c80231 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:23:37 +0100 Subject: [PATCH 2172/2779] updated README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index ec5ba95b9..908e5d1b1 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,12 @@ Hari Sekhon - DevOps Python Tools [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) +[![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) +[![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) +[![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) +[![Docker Build (Fedora)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml) +[![Docker Build (Ubuntu)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml) + [![GitHub Actions Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/GitHub%20Actions%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22) [![Mac](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Mac/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac%22) [![Mac 10.15](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Mac%2010.15/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac+10.15%22) From 7dae166c5609f775c8a60e77a1d7fc166d4fc3c2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:40:23 +0100 Subject: [PATCH 2173/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1f9ee12e5..4f58bb28f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1f9ee12e52063c59c0620842bfe7186e91bec5a8 +Subproject commit 4f58bb28f8f2d16cadbf921522ccc044d643728b From d1d84c8a4f2e24d005c199aee8768dfdd6d2e005 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:40:23 +0100 Subject: [PATCH 2174/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index aec17428f..85ed84469 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit aec17428f1a946d4b2ead6b37379b84c9b625a33 +Subproject commit 85ed84469db95f2e66b5f1e3d0a044a80a49885f From 317a75c51d787d24f0a590a550ad5b09a843ae1f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:40:23 +0100 Subject: [PATCH 2175/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 69aff25cb..0c8255344 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 69aff25cbac90c8fc97a66c0d9e4351ea659c54e +Subproject commit 0c825534472969e6a41770e9246ede7861dbbb10 From 89766f027592e0e3d657e37261a6c27abf77b508 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 6 Apr 2022 19:40:23 +0100 Subject: [PATCH 2176/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index e444f83ec..3417faa70 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit e444f83ecb0082c0639c4df23ca24e7790955d7b +Subproject commit 3417faa7035c29de864f101c730a01176f563f38 From 9d5f41536225ef8e93b644898119b5347e273b27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Apr 2022 22:27:27 +0100 Subject: [PATCH 2177/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 908e5d1b1..57a606b7b 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Hari Sekhon - DevOps Python Tools [![MicroBadger](https://images.microbadger.com/badges/image/harisekhon/pytools.svg)](http://microbadger.com/#/images/harisekhon/pytools) --> -[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md) +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://harisekhon.github.io/CI-CD/) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) [![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) [![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) From cdaa1d3707cb8593421091ac92a0a74cf31c81ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Apr 2022 22:30:29 +0100 Subject: [PATCH 2178/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4f58bb28f..07871b12f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4f58bb28f8f2d16cadbf921522ccc044d643728b +Subproject commit 07871b12f83f5870d7ada82fa3de8b43ed0b56d4 From 945574937e48a2554487e1324c1953b174cde83d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Apr 2022 22:30:29 +0100 Subject: [PATCH 2179/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 85ed84469..965905dd6 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 85ed84469db95f2e66b5f1e3d0a044a80a49885f +Subproject commit 965905dd62b8c402f36a546c043fad07d4ae9d92 From 452d52c2f0c6d4650061485e653e1c0a5965a985 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 7 Apr 2022 22:30:30 +0100 Subject: [PATCH 2180/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3417faa70..e116df4dd 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3417faa7035c29de864f101c730a01176f563f38 +Subproject commit e116df4ddff7d917fa28291cbca1f1744c8bc664 From 22760c05091ab30ac7d394214904e1d1db7a625b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 10:36:57 +0100 Subject: [PATCH 2181/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57a606b7b..b972256b0 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Hari Sekhon - DevOps Python Tools [![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml) -[![Travis CI](https://img.shields.io/badge/TravisCI-legacy-lightgrey?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) +[![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) From a26a0cc966cb71d66ecc03d7300e69077e688110 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:12 +0100 Subject: [PATCH 2182/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index 287b2d490..a2dfe24e2 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From d183af7d3fe3279cccfe876a19c12268963b5067 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:13 +0100 Subject: [PATCH 2183/2779] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 96f31e3ab..2a7fbaff0 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From b9127c3d25bf64329361d9bddd88adb95c6aaea0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:14 +0100 Subject: [PATCH 2184/2779] updated centos.yaml --- .github/workflows/centos.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index ed6372ec7..fbaf19c4f 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 56ed0ad957eebdf7931345efe48cec969f552bdf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:14 +0100 Subject: [PATCH 2185/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 681be974f..ebaf5fae2 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 007cf727eb4d8bd6b9c1ca30286385675463b061 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:14 +0100 Subject: [PATCH 2186/2779] updated centos8.yaml --- .github/workflows/centos8.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 9c4311285..fada215de 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From c931a9e1bc77899cc161faea6074940022e52ec8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:15 +0100 Subject: [PATCH 2187/2779] updated checkov.yaml --- .github/workflows/checkov.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index d05b53148..26be616d6 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -27,10 +27,14 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' pull_request: branches: - master - main + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 21daa159117d7b16b845fcb2bfe9c2459874a64b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:15 +0100 Subject: [PATCH 2188/2779] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index d4e575f30..934d09b92 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -19,10 +19,16 @@ on: branches: - master - main + paths: + - CODEOWNERS + - .github/CODEOWNERS pull_request: branches: - master - main + paths: + - CODEOWNERS + - .github/CODEOWNERS workflow_dispatch: inputs: debug: From 7656bda959896e8b68d1b45ba321b518efeb7008 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:15 +0100 Subject: [PATCH 2189/2779] updated debian.yaml --- .github/workflows/debian.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index ae2afe46e..e993c7222 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 9a5c9408ed80b2b874c75589e49ea33edab99248 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:16 +0100 Subject: [PATCH 2190/2779] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 2656f55fd..b120564e9 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From b8e0debfd788747913737985f936a6b1763127b2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:16 +0100 Subject: [PATCH 2191/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index dff981bd2..2e8c06e55 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From ddcbd8312c126630c576b86cfc9fdf2302507fb4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:16 +0100 Subject: [PATCH 2192/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 74b06b44e..65fe55cca 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 8e76ebbe7956dfd59daaebf459ed25f2ea34f40d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:17 +0100 Subject: [PATCH 2193/2779] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 6b2a6f979..ae2bc9f9b 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From d4f65ff7955209a09a20185f32b192b00f2f819e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:17 +0100 Subject: [PATCH 2194/2779] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 3641c892c..9223474c1 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From a823e30859dbee0eb6d60b61c5c8ccc15940c961 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:17 +0100 Subject: [PATCH 2195/2779] updated fedora.yaml --- .github/workflows/fedora.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 2128939b8..fe6bcbe6f 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 4d496a1955bfa19a95949f22df958272b346ac91 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:18 +0100 Subject: [PATCH 2196/2779] updated json.yaml --- .github/workflows/json.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index b66e8cee6..5a9df06b5 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -19,10 +19,14 @@ on: branches: - master - main + paths: + - '**/*.json' pull_request: branches: - master - main + paths: + - '**/*.json' workflow_dispatch: inputs: debug: From 19cf93bff84c4f0befe5657cece5530619581e16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:18 +0100 Subject: [PATCH 2197/2779] updated mac.yaml --- .github/workflows/mac.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 81aaec5b1..92e326bcb 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 2bae838d15d705147545257da15ccb89b14815cb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:19 +0100 Subject: [PATCH 2198/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index db4781fab..b8feb60c0 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 61d92389ae1a26f11e498a7d765705cd02f4cd43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:19 +0100 Subject: [PATCH 2199/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index ae3774daf..707dd82f0 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -17,6 +17,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 531040f2624195c209434d5652a9ea626d4e933d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:20 +0100 Subject: [PATCH 2200/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 966741006..c379246c9 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -17,6 +17,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 08a217b29f5d3cfa77fe45e6f4d0d2016e830ecd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:20 +0100 Subject: [PATCH 2201/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index bb03008d4..a850b2b84 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From d1cf2e7c7c99f81af2c791d7ed46d406dbd49be5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:20 +0100 Subject: [PATCH 2202/2779] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 4ba2d2cdf..da5b11f35 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 65919da01b1375e3228520ca50a2b645cb3eed26 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:21 +0100 Subject: [PATCH 2203/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index d94daf551..32e3089e2 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 636e1258db0c350109ec72fa3ba70de681a70f42 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:21 +0100 Subject: [PATCH 2204/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 9e2167f91..d6efcf883 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 9db851fdc24075bf69f22df58bf862bc20bd13f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:21 +0100 Subject: [PATCH 2205/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 0adb02bb5..f0eb8fa2b 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From 21f40ffdd707e62456771c7a39509cd41e5e899c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:22 +0100 Subject: [PATCH 2206/2779] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 04a325e29..df374c960 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -18,6 +18,12 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' workflow_dispatch: inputs: debug: From d4ba19a16ee1346e6b1856cb44acff99da51b5d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:22 +0100 Subject: [PATCH 2207/2779] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 1e8278f7e..47fe5fcf5 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -25,10 +25,14 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' pull_request: branches: - master - main + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 374acc85391a98039f23afd172bf6f6b8fad7128 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:22 +0100 Subject: [PATCH 2208/2779] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 410e63c0f..ea33c7f40 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -27,10 +27,14 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' pull_request: branches: - master - main + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 541276e36f87f19734af2300e0bc98f74a331ff0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:23 +0100 Subject: [PATCH 2209/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 927f7edc2..6a623c6da 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From cd2ef8bfae5853d3363be2cc7d5af6ff0423b87a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:23 +0100 Subject: [PATCH 2210/2779] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index fff40eaa6..915f04ae9 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From c2f35969f7ee49177cf2b697c532cce0bc811824 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:23 +0100 Subject: [PATCH 2211/2779] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index cd96b0858..f459f4398 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From df957eaa9e121ab791f9f61801390b173207998b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:24 +0100 Subject: [PATCH 2212/2779] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 385bcfff6..926c7e0a5 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 8370c6011bcdec7d2cee8c5455ad23dd4a033031 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:24 +0100 Subject: [PATCH 2213/2779] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 360a8db4e..ad419846f 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 654d37ca583fa97480860ab36c5b5d7ad2eee017 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:25 +0100 Subject: [PATCH 2214/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 59a9c9110..85bb7dd48 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -18,6 +18,8 @@ on: push: branches: - master + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 520778e57013988327c8c298cff3be492e88909d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:25 +0100 Subject: [PATCH 2215/2779] updated validate.yaml --- .github/workflows/validate.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index bbdc02826..b9634e4ae 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -19,10 +19,14 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' pull_request: branches: - master - main + paths-ignore: + - '**/*.md' workflow_dispatch: inputs: debug: From 5862e20bb9cf0e6c8e3d027a0d721f9f33caa96f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:23:25 +0100 Subject: [PATCH 2216/2779] updated yaml.yaml --- .github/workflows/yaml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index c11764e11..42f1aae0e 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -19,10 +19,16 @@ on: branches: - master - main + paths: + - '**/*.yml' + - '**/*.yaml' pull_request: branches: - master - main + paths: + - '**/*.yml' + - '**/*.yaml' workflow_dispatch: inputs: debug: From cad5cf8bef2f789d86619ee6ea6c2c52fd1baa3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 14:26:38 +0100 Subject: [PATCH 2217/2779] renamed ghcr_python_ubuntu.yaml to ghcr_python_ubuntu.yaml.disabled --- .../{ghcr_python_ubuntu.yaml => ghcr_python_ubuntu.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ghcr_python_ubuntu.yaml => ghcr_python_ubuntu.yaml.disabled} (100%) diff --git a/.github/workflows/ghcr_python_ubuntu.yaml b/.github/workflows/ghcr_python_ubuntu.yaml.disabled similarity index 100% rename from .github/workflows/ghcr_python_ubuntu.yaml rename to .github/workflows/ghcr_python_ubuntu.yaml.disabled From be9fcb18e46359a1befced1c8f96ef9d934ad53f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:39 +0100 Subject: [PATCH 2218/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a2dfe24e2..c07877011 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:latest From 002e00ed5fbd3d40ac861b88d7bfaf338735cf2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:39 +0100 Subject: [PATCH 2219/2779] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index 2a7fbaff0..e692d25b1 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: alpine:3 From 20a03a92addf21803a0c03bcf21f40b1e8bfd89e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:39 +0100 Subject: [PATCH 2220/2779] updated centos.yaml --- .github/workflows/centos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index fbaf19c4f..d4147ea01 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:latest From 211ed21ca520d9754d2a4f018348db986f3141f6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:39 +0100 Subject: [PATCH 2221/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index ebaf5fae2..5a588f28d 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:7 From 024e14c8e2c24e1645ac3d34de9afc9135f71a69 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 2222/2779] updated centos8.yaml --- .github/workflows/centos8.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index fada215de..56fdbba13 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: centos:8 From ea3718ed1bc35ddf15dfe757ee6096bbdac46fd5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 2223/2779] updated debian.yaml --- .github/workflows/debian.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index e993c7222..98f6d7414 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:latest From dae14434eecd8f00ed847f645aa559dcd95f07da Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 2224/2779] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index b120564e9..b45963ac3 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:10 From cb0fd02a788b269b924d816c226827c74a2ae4e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 2225/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 2e8c06e55..bcfb2efca 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:6 From 3b81f25ffc19f84aa4f4b902acca14d82f7ef399 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:40 +0100 Subject: [PATCH 2226/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index 65fe55cca..f09fdf524 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:7 From c4930b32f7160b574b5133118fd479cd8c64f15a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 2227/2779] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index ae2bc9f9b..043bc389b 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:8 From ac421dffa9eabf26f492a3d84610ead2627507fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 2228/2779] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 9223474c1..6503faeeb 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: debian:9 From 737e740ec8b3e50ad90eb9be15e57aff3944dddc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 2229/2779] updated fedora.yaml --- .github/workflows/fedora.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index fe6bcbe6f..71d54f26a 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: fedora From 490e2a8ef76bbdab824ab66a2a69d293a13683c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 2230/2779] updated mac.yaml --- .github/workflows/mac.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 92e326bcb..fef39007c 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: runs-on: macos-latest From 542ec9e11554d900442ace3251d2e09f516e7ca0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 2231/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index b8feb60c0..fcc669e97 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: runs-on: macos-10.15 From 48cb53b8f6da9cefd0d96c659f835a05254984f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:41 +0100 Subject: [PATCH 2232/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index 707dd82f0..b6436c968 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -11,12 +11,24 @@ # https://www.linkedin.com/in/HariSekhon # +--- name: PyPy 2 on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -43,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: PyPy2 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: pypy2 + python-version: pypy2 debug: ${{ github.event.inputs.debug }} From 4d23cfcbc9d9d0b85b36aa2b1d89260824f0eca1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 2233/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index c379246c9..6c7d2a402 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -11,12 +11,24 @@ # https://www.linkedin.com/in/HariSekhon # +--- name: PyPy 3 on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -43,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: PyPy3 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: pypy3 + python-version: pypy3 debug: ${{ github.event.inputs.debug }} From 8de68c7f994ed783f09f88d23a10b6055dd9121c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 2234/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index a850b2b84..727e300f0 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 2.7 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 2.7 + python-version: 2.7 debug: ${{ github.event.inputs.debug }} From 92bb5125370cbf97d08ac5f237a9bd97f142df5d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 2235/2779] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index da5b11f35..31bf6e30c 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.10 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: "3.10" + python-version: "3.10" debug: ${{ github.event.inputs.debug }} From 20ec0d435277be90e08b840163fcc8485a7b9a0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 2236/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 32e3089e2..3bb00c01d 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.6 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3.6 + python-version: 3.6 debug: ${{ github.event.inputs.debug }} From 004bf873069e33a4748a44835a57d9a5ec404f99 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:42 +0100 Subject: [PATCH 2237/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index d6efcf883..0d463f1f0 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.7 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3.7 + python-version: 3.7 debug: ${{ github.event.inputs.debug }} From c920da17656d53446508dab674afdcae8a358ffc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 2238/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index f0eb8fa2b..4a1df9f81 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.8 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3.8 + python-version: 3.8 debug: ${{ github.event.inputs.debug }} From c6236581ffe418815c6157e7a7884e079da991eb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 2239/2779] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index df374c960..e4aa45470 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -18,6 +18,17 @@ on: push: branches: - master + - main + paths-ignore: + - '**/*.md' + - '**/*.pl' + - '**/*.rb' + - '**/*.go' + - '**/*.sh' + pull_request: + branches: + - master + - main paths-ignore: - '**/*.md' - '**/*.pl' @@ -44,7 +55,7 @@ jobs: build: if: github.event.repository.fork == false name: Python 3.9 - uses: HariSekhon/GitHub-Actions/.github/workflows/python.yaml@master + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: - version: 3.9 + python-version: 3.9 debug: ${{ github.event.inputs.debug }} From a19439b6e56fd29c84334552ebdd1c30e7362c0d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 2240/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 6a623c6da..b7d2f2e1c 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:latest From 9a67e78b8e7e703ab60fdddb9794cff899391c30 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 2241/2779] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index 915f04ae9..b38f1bfcd 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:14.04 From a4ee6d97c9f1c7366ffadea18e1ba8a5e6b6c1ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 2242/2779] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index f459f4398..26802e404 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:16.04 From 46f1972f7732ec55e4d04216653bdf1ad9a8629a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:43 +0100 Subject: [PATCH 2243/2779] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 926c7e0a5..70bc87eb6 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:18.04 From 50e84e49625b5481987bfa49f0dbaacc8247bcd7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:44 +0100 Subject: [PATCH 2244/2779] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index ad419846f..46fa80dc6 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: container: ubuntu:20.04 From 8a32413be768268fa1f469fc419199feb8448674 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:44:44 +0100 Subject: [PATCH 2245/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 85bb7dd48..5b736dffc 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -37,9 +37,9 @@ concurrency: cancel-in-progress: true jobs: - make: + build: if: github.event.repository.fork == false - name: Make + name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: caches: apt pip cpanm From 22b6a697b212e0a4528d7c4b69556c1f8c121117 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:49:17 +0100 Subject: [PATCH 2246/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 07871b12f..9e7f34224 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 07871b12f83f5870d7ada82fa3de8b43ed0b56d4 +Subproject commit 9e7f342243b5514ed0d9b3e04228538d2ee98330 From c3a6cf2d9ce61ff83222f900aaf7722f2e9b8f37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:49:17 +0100 Subject: [PATCH 2247/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 965905dd6..d23609649 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 965905dd62b8c402f36a546c043fad07d4ae9d92 +Subproject commit d2360964912b643102ac73433cbbe8cc542d4c40 From 58ae9294804702179e0cff0d31c8a1c6cb969700 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:49:17 +0100 Subject: [PATCH 2248/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 0c8255344..282fb3eb0 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 0c825534472969e6a41770e9246ede7861dbbb10 +Subproject commit 282fb3eb06ee967f0bc308b4506b5d72d8cbccd0 From a886e32a52c31c0e863db35095d39f321c6fbb8f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:49:17 +0100 Subject: [PATCH 2249/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index e116df4dd..94b23ae15 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit e116df4ddff7d917fa28291cbca1f1744c8bc664 +Subproject commit 94b23ae1575080d1771f67bf0f3ee4e0f490b99d From 83bedbaa955264ce3502b29ae53e07ac61596aca Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:57 +0100 Subject: [PATCH 2250/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index b6436c968..d21626a61 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: pypy2 + caches: apt pip debug: ${{ github.event.inputs.debug }} From 420f6e3159ab1ce6a2cfe72a0bff6e0d4afadcfe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:57 +0100 Subject: [PATCH 2251/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 6c7d2a402..9c558f16a 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: pypy3 + caches: apt pip debug: ${{ github.event.inputs.debug }} From faae953eb96cafa4af86ada0ecf4bc08608674bd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:57 +0100 Subject: [PATCH 2252/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 727e300f0..707543308 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 2.7 + caches: apt pip debug: ${{ github.event.inputs.debug }} From 3758fbb2015b2822587b5a8bc4570eea4781eb44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 2253/2779] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 31bf6e30c..fda309782 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: "3.10" + caches: apt pip debug: ${{ github.event.inputs.debug }} From 69da3a6fb05a1d3fea8e3b99e70bc61cec19cd0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 2254/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 3bb00c01d..84c61bc5f 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.6 + caches: apt pip debug: ${{ github.event.inputs.debug }} From b68c897a33154a98c032f03e340caaaa6fda0246 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 2255/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 0d463f1f0..6a4191bcc 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.7 + caches: apt pip debug: ${{ github.event.inputs.debug }} From d099da0e894e4ef138049ef975be6a7279023059 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 2256/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 4a1df9f81..de4668018 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.8 + caches: apt pip debug: ${{ github.event.inputs.debug }} From 337bb7202dfed79d5e35d4b8543ed55e31ff09b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:55:58 +0100 Subject: [PATCH 2257/2779] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index e4aa45470..97914800c 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -58,4 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: python-version: 3.9 + caches: apt pip debug: ${{ github.event.inputs.debug }} From d70b960e126ee5948f8e7b3425a0a2956368b6e8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:58:13 +0100 Subject: [PATCH 2258/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9e7f34224..fc1c416f5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9e7f342243b5514ed0d9b3e04228538d2ee98330 +Subproject commit fc1c416f5a808fc5bf42b369c46e70ee3410286e From 937b9981b9663a090a6640057b089a0cf22c7b60 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:58:13 +0100 Subject: [PATCH 2259/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d23609649..3bf8af87c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d2360964912b643102ac73433cbbe8cc542d4c40 +Subproject commit 3bf8af87c27ed8e9babef1bbd5c414aa0678e021 From 43fd53a7c4dd592c7e2d10b3acc839e72e3ecc4e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 15:58:13 +0100 Subject: [PATCH 2260/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 94b23ae15..26c0a409b 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 94b23ae1575080d1771f67bf0f3ee4e0f490b99d +Subproject commit 26c0a409b8d18d3fdea3eb68db5d8c8af87908f7 From 04c3bb05d4bb816769795536bd137752efc36dcc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 18:41:22 +0100 Subject: [PATCH 2261/2779] updated README.md --- README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b972256b0..3aa702615 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,11 @@ Hari Sekhon - DevOps Python Tools ================================= -[![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) -[![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) -[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) -[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/network) -[![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) +[![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) +[![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) @@ -24,6 +15,17 @@ Hari Sekhon - DevOps Python Tools [![Python 3](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/python-3-shield.svg)](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/) --> +[![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) +[![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) +[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) +[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) +[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) + [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) [![Docker](https://img.shields.io/badge/container-Docker-blue?logo=docker&logoColor=white)](https://hub.docker.com/r/harisekhon/github/) From e332dd04dd88f32c49b75e9b8962c4537472c929 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:23:28 +0100 Subject: [PATCH 2262/2779] added .werckerignore --- .werckerignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .werckerignore diff --git a/.werckerignore b/.werckerignore new file mode 100644 index 000000000..ff8fb8247 --- /dev/null +++ b/.werckerignore @@ -0,0 +1 @@ +**/.md From 73c1f49ab95c412844cafa43cd4971e23cd227a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:24:53 +0100 Subject: [PATCH 2263/2779] updated .cirrus.yml --- .cirrus.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cirrus.yml b/.cirrus.yml index 46de8ac49..7ea506862 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -19,6 +19,7 @@ container: image: ubuntu:18.04 task: + skip: "!changesInclude('**/*.md')" script: - setup/ci_bootstrap.sh - make init From 6675defaaecc6592ebe734de7f6ee91e81a86464 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:25:30 +0100 Subject: [PATCH 2264/2779] updated .appveyor.yml --- .appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index f23536e32..1551a3473 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -17,6 +17,11 @@ image: Ubuntu +skip_commits: + files: + - docs/* + - '**/*.md' + # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ environment: APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension From a5b27a7d95d37062038dabd5c300afd6b39baee3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:26:08 +0100 Subject: [PATCH 2265/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 6e5dc9f25..99040a203 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -26,7 +26,7 @@ execution_time_limit: blocks: - name: Linux build run: - when: "branch = 'master'" + when: "branch = 'master' AND change_in('/', {exclude: ['**/*.md']})" #execution_time_limit: # hours: 2 task: From 33613dcb26325f085651eb7aa1f84517c65aac32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:35:27 +0100 Subject: [PATCH 2266/2779] updated .cirrus.yml --- .cirrus.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 7ea506862..7f1425a4c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -19,7 +19,8 @@ container: image: ubuntu:18.04 task: - skip: "!changesInclude('**/*.md')" + # doesn't work properly + #skip: "!changesInclude('**/*.md')" script: - setup/ci_bootstrap.sh - make init From 54a6e3b9b8a94ee0161a84ebb3fdd7aafc9f0855 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:38:15 +0100 Subject: [PATCH 2267/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fc1c416f5..9d6404984 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fc1c416f5a808fc5bf42b369c46e70ee3410286e +Subproject commit 9d640498465c747e94e8e2e281e252af2cf3096f From cdaef5573a767373b4e49dd36e583c83c462b4e0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:38:15 +0100 Subject: [PATCH 2268/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 3bf8af87c..c7887c4bb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 3bf8af87c27ed8e9babef1bbd5c414aa0678e021 +Subproject commit c7887c4bbce54d0cbc2dec40676fcf39209ad992 From fbcbb926aea17b127c413706e041e05c6a312473 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:38:15 +0100 Subject: [PATCH 2269/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 282fb3eb0..fb98e0a2f 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 282fb3eb06ee967f0bc308b4506b5d72d8cbccd0 +Subproject commit fb98e0a2fa84f94916d02336d9eee323e96e34b9 From 3fcc6510f18532fbf15dffd8d20cbb299728f638 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 8 Apr 2022 19:38:15 +0100 Subject: [PATCH 2270/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 26c0a409b..9a8db2f6e 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 26c0a409b8d18d3fdea3eb68db5d8c8af87908f7 +Subproject commit 9a8db2f6e480766c35ad3332171cfa0aaa697d1a From 615319007021fd79410ffb215bcb65a7c62baefb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 11 Apr 2022 15:58:48 +0100 Subject: [PATCH 2271/2779] updated Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 66b95292f..b7f961b67 100644 --- a/Makefile +++ b/Makefile @@ -210,7 +210,8 @@ test-lib: cd pylib && $(MAKE) test .PHONY: test -test: test-lib +#test: test-lib +test: tests/all.sh .PHONY: basic-test From 1a9f9b484f6485ac508868005e9e7239f49ad825 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:39:23 +0100 Subject: [PATCH 2272/2779] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 556bd80d9..cad44a536 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -22,7 +22,7 @@ on: required: false default: false schedule: - - cron: '0 9 * * *' + - cron: '30 9 * * 2' permissions: contents: write From 1099646495a4a0779ac722f18fd52e2ac4acde67 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:39:24 +0100 Subject: [PATCH 2273/2779] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 12449fb36..1897533a2 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,8 +22,8 @@ on: required: false default: false schedule: - # fork-sync happens for default branch every hour, so just after that, run PRs for branches - - cron: '2 10 * * 2' + # fork-sync happens 9:30 every tuesday, so after that run PRs for branches + - cron: '0 10 * * 2' permissions: contents: write From b6161858a07af7d574f5bf8c3c32eaa70cadd34e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:44:10 +0100 Subject: [PATCH 2274/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9d6404984..62a152225 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9d640498465c747e94e8e2e281e252af2cf3096f +Subproject commit 62a152225697e0c05244aa52ab875dc2978baf20 From 6316b262c9cbda7f73db4a57112eaf5adce021ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:44:10 +0100 Subject: [PATCH 2275/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c7887c4bb..8ae4d15e2 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c7887c4bbce54d0cbc2dec40676fcf39209ad992 +Subproject commit 8ae4d15e20b0f061385485abea2369e2b3e634e6 From 6cbd8eaea51bd48bb4865c9f3e68a8dab207dc82 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:44:10 +0100 Subject: [PATCH 2276/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index fb98e0a2f..6f5a26d58 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit fb98e0a2fa84f94916d02336d9eee323e96e34b9 +Subproject commit 6f5a26d58504258b2749a8316ae92b3836d27bf8 From 6874f264b882f5218d5002d8e67ee09aa8829f10 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 16:44:10 +0100 Subject: [PATCH 2277/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9a8db2f6e..68441f7ed 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9a8db2f6e480766c35ad3332171cfa0aaa697d1a +Subproject commit 68441f7edeeedd5408c95e3969eaa0f3721b5296 From afaabdab1a225543a8a1ed5d183debc3e456b312 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 2278/2779] updated alpine.yaml --- .github/workflows/alpine.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index c07877011..39e4a4497 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 83d72dd0ebc14e8a3c59d6d99af36f33b09d6e1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 2279/2779] updated alpine_3.yaml --- .github/workflows/alpine_3.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index e692d25b1..0b1cbea05 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 4ed362a170e37faa5313e28b30019e9e623cb576 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 2280/2779] updated centos.yaml --- .github/workflows/centos.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml index d4147ea01..6f3dd2b27 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From f820794229afbb69248abbef61b1857162052f16 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 2281/2779] updated centos7.yaml --- .github/workflows/centos7.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml index 5a588f28d..46c3ca212 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From a28ea1564a505b0ae01555f5e85215215ba02170 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:39 +0100 Subject: [PATCH 2282/2779] updated centos8.yaml --- .github/workflows/centos8.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml index 56fdbba13..782ab12ff 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 13c981adaa13d0fd56cd93a313da5e68c96e86b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 2283/2779] updated checkov.yaml --- .github/workflows/checkov.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 26be616d6..8f31b2672 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -51,7 +51,9 @@ permissions: jobs: checkov: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Checkov uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master with: From 0122b4e055177136184ebafc95fccf2929eafbcf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 2284/2779] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 934d09b92..5fdb3401e 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -47,6 +47,8 @@ concurrency: jobs: validate: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Validate CODEOWNERS uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master From a44c53e059acecf61c0f4534fb5b9c8b32b1847e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 2285/2779] updated debian.yaml --- .github/workflows/debian.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 98f6d7414..6ded416ad 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 47a6dd8652512e6409d43862f934431f8fd70993 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 2286/2779] updated debian_10.yaml --- .github/workflows/debian_10.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index b45963ac3..debc3945d 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 913f5c50420810c7289add8da7610690e6eefad0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 2287/2779] updated debian_6.yaml.disabled --- .github/workflows/debian_6.yaml.disabled | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index bcfb2efca..1cb4e5431 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 6eb8c5a5a2188ad2b331c70fe0344a7cbacd3702 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 2288/2779] updated debian_7.yaml.disabled --- .github/workflows/debian_7.yaml.disabled | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index f09fdf524..e987d7e46 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 45bc93d322d763fb16fa99cf07686ee70a6e98e6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 2289/2779] updated debian_8.yaml --- .github/workflows/debian_8.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml index 043bc389b..e8fc5595a 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 04c216e6037e3558ff2ae3be261d627b7d1d349f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 2290/2779] updated debian_9.yaml --- .github/workflows/debian_9.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml index 6503faeeb..5c14364d9 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 000a39bda485bf53f61c4f92573caff780115946 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:40 +0100 Subject: [PATCH 2291/2779] updated fedora.yaml --- .github/workflows/fedora.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 71d54f26a..0c29aeccb 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 615ae322dbb8ec544ef53e584c0eab2e2fb5aa2a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:41 +0100 Subject: [PATCH 2292/2779] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index cad44a536..f1e605236 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -33,6 +33,8 @@ concurrency: jobs: fork_sync: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == true if: github.repository_owner != 'HariSekhon' name: Fork Sync uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master From 61cb4a4f7a7f81ea7c16769d6fe84f512be378f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:41 +0100 Subject: [PATCH 2293/2779] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 1897533a2..1767b3e33 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -35,6 +35,8 @@ concurrency: jobs: fork_update_pr: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == true if: github.repository_owner != 'HariSekhon' name: Fork Update PR uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master From d03abe30fdf50b23c05b8276d9cc3f25df81c1e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:41 +0100 Subject: [PATCH 2294/2779] updated json.yaml --- .github/workflows/json.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index 5a9df06b5..5d70a6e2a 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -41,6 +41,8 @@ permissions: jobs: check_json: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Check JSON uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master From b42654902236e939461b57f76655bc185a061589 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:41 +0100 Subject: [PATCH 2295/2779] updated mac.yaml --- .github/workflows/mac.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index fef39007c..06e696b85 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 367958487167f57e5a1070c7de6a597e218c66c4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2296/2779] updated mac_10.15.yaml --- .github/workflows/mac_10.15.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml index fcc669e97..4f42839e7 100644 --- a/.github/workflows/mac_10.15.yaml +++ b/.github/workflows/mac_10.15.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 3d33c7d16aea8d3d7e6780959b26b1a6187d0d95 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2297/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index d21626a61..ff7ebfc1f 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: PyPy2 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 0a66d7b7646a10df5a2e9c852e7015e33ca2e777 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2298/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index 9c558f16a..be648b6b7 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: PyPy3 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From a93c9abd53b99757035206d12eaf6bfec5f84cdf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2299/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 707543308..6f6a3cc13 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 2.7 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 352a8ad47386e38b2771bb75df4d0f23a39c0880 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2300/2779] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index fda309782..c7710cc6d 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.10 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 38b2abd4f786a6623b4272cc6bbaf9cb6b63ee29 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2301/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index 84c61bc5f..a37f4b538 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.6 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 4605089e36a3d17be5b4e9d805fcec38326296e7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2302/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 6a4191bcc..8c19509a2 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.7 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 4cd40000afa4f288642c08881c98b3a7a8417c70 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2303/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index de4668018..025424851 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.8 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 1e8f850467fc1e016ae78214be928ed74916b7ae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2304/2779] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 97914800c..862e2eccf 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -53,7 +53,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Python 3.9 uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From e1599fcee8a4ea5b9994bfe2db2216038767ec3d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2305/2779] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 47fe5fcf5..9bb52df1a 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -51,7 +51,9 @@ concurrency: jobs: semgrep: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Semgrep Cloud uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: From 26983c4b3328b23e0f7dece4e4e4399ef87c831b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2306/2779] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index ea33c7f40..cacc83e09 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -55,6 +55,8 @@ concurrency: jobs: semgrep: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master From db5dc7a745c81da469a2a21831c4909095db5a09 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:42 +0100 Subject: [PATCH 2307/2779] updated ubuntu.yaml --- .github/workflows/ubuntu.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index b7d2f2e1c..d6acb8902 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 81a4ff08c879b0f243c9a77a95164bae199c46bb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 2308/2779] updated ubuntu_14.04.yaml --- .github/workflows/ubuntu_14.04.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml index b38f1bfcd..da84dee9c 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 2745c6081b0003bc43c800c77d337c3e278df4aa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 2309/2779] updated ubuntu_16.04.yaml --- .github/workflows/ubuntu_16.04.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml index 26802e404..a77170ded 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 08c56418776425a67ae79de8ff474c7fd9748e06 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 2310/2779] updated ubuntu_18.04.yaml --- .github/workflows/ubuntu_18.04.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml index 70bc87eb6..5e66ef8d3 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 19f3b31a1875a5cc11b7d69a9e6307998306adec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 2311/2779] updated ubuntu_20.04.yaml --- .github/workflows/ubuntu_20.04.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 46fa80dc6..7da14d8bc 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From acfddac39d122371223af906eeb7f5fdbaa489c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 2312/2779] updated ubuntu_github.yaml --- .github/workflows/ubuntu_github.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 5b736dffc..5a24bd177 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -38,7 +38,9 @@ concurrency: jobs: build: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Build uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master with: From 21adc74146717bbeca517c5e4856ec749a201b2a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 2313/2779] updated validate.yaml --- .github/workflows/validate.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index b9634e4ae..a2daad965 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -45,6 +45,8 @@ concurrency: jobs: validate: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Validate uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master From 7767f1c7d3a77a1094c1d5d476b9fbca8eca70c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 17:58:43 +0100 Subject: [PATCH 2314/2779] updated yaml.yaml --- .github/workflows/yaml.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index 42f1aae0e..81a6578a6 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -47,6 +47,8 @@ concurrency: jobs: check_yaml: - if: github.event.repository.fork == false + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' name: Check YAML uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master From 6b71f80bd0b4c07e0a54f46a3219e993ac2fe9fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 19:32:08 +0100 Subject: [PATCH 2315/2779] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index f1e605236..4d4ebd577 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -22,7 +22,7 @@ on: required: false default: false schedule: - - cron: '30 9 * * 2' + - cron: '0 */3 * * *' permissions: contents: write From 00a4cadd8711a2ad22e4e43f89cc3eddb1ff9335 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 21 Apr 2022 19:32:08 +0100 Subject: [PATCH 2316/2779] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 1767b3e33..eff67422c 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,7 +22,6 @@ on: required: false default: false schedule: - # fork-sync happens 9:30 every tuesday, so after that run PRs for branches - cron: '0 10 * * 2' permissions: From 899f6498e290a1c65f8ec31e33a0a6ea34a4a712 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Apr 2022 12:34:44 +0100 Subject: [PATCH 2317/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 62a152225..c9878cdf5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 62a152225697e0c05244aa52ab875dc2978baf20 +Subproject commit c9878cdf5cb52eea7366c73967e415a2a25cb62a From d7678a89275e50e4022eff301e27cbc20dd3c1a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Apr 2022 12:34:44 +0100 Subject: [PATCH 2318/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 8ae4d15e2..1dc4ec7c5 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 8ae4d15e20b0f061385485abea2369e2b3e634e6 +Subproject commit 1dc4ec7c57ea82317910cb6d9a550837af120a68 From c749b62f069144e277b62a12e0a0e4a3ed98b2d0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Apr 2022 12:34:44 +0100 Subject: [PATCH 2319/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 6f5a26d58..9ac92d3dc 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 6f5a26d58504258b2749a8316ae92b3836d27bf8 +Subproject commit 9ac92d3dcb3c2f72d0b9ab896604e932f5d43aa7 From 17c617a928901140f5262de94b77ee1b5188d815 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Apr 2022 12:34:44 +0100 Subject: [PATCH 2320/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 68441f7ed..7330e7ab9 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 68441f7edeeedd5408c95e3969eaa0f3721b5296 +Subproject commit 7330e7ab958a08a3f22a70dfdb4d807dc9276d74 From 14923853f2771751c179cd867a0d94b9ea418bac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 18:44:33 +0100 Subject: [PATCH 2321/2779] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index eff67422c..00479ecb7 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,7 +22,7 @@ on: required: false default: false schedule: - - cron: '0 10 * * 2' + - cron: '0 10 * * 1' permissions: contents: write From af2e0f141749df2aeac0f488ebbd129453d5ecdc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 2322/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c9878cdf5..4bc47ad4e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c9878cdf5cb52eea7366c73967e415a2a25cb62a +Subproject commit 4bc47ad4e3dc0d23f08054b6f4687a92869ec9f5 From 1999b1bbabccf27c89a7ea9c06dfaff42bce782d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 2323/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1dc4ec7c5..b22457e26 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1dc4ec7c57ea82317910cb6d9a550837af120a68 +Subproject commit b22457e26204158f537524111be4e45669b82663 From 504eb8b6c3a39f38e2b683a9328e988dd4da7a77 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 2324/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 9ac92d3dc..300fd2a80 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 9ac92d3dcb3c2f72d0b9ab896604e932f5d43aa7 +Subproject commit 300fd2a80741eaf8dcf3e2900f507a5531355bd7 From 749bd6a88cd56ff6c69c59d0aa0e44e282f6b8ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 2325/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 7330e7ab9..9a3f5c616 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 7330e7ab958a08a3f22a70dfdb4d807dc9276d74 +Subproject commit 9a3f5c616667cab76f98c13f2d8576610b877d1f From ed6feacd0214d212eaa0eb96a55fecde0798809b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:27 +0100 Subject: [PATCH 2326/2779] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index ff7ebfc1f..7dbc25c7f 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 3e3a05b93f242a81ae2943d73087e9ff2085e529 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:27 +0100 Subject: [PATCH 2327/2779] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index be648b6b7..ac669813c 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 058b98407d0b1b131ba6e9ce01ea2059c5a6ef0c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:27 +0100 Subject: [PATCH 2328/2779] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 6f6a3cc13..60b31b6bc 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 973c93960cd0793efa8f9dd739c4c5249ae8ea66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 2329/2779] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index c7710cc6d..939ae7337 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 6516c90d6b7eb00bc64f96b00fb14acd049f0b4d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 2330/2779] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index a37f4b538..d6a61ea45 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 6bcf6787f950225bf13bce535cd23accce2a1f62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 2331/2779] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 8c19509a2..70ca1f828 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From edb68351bd5881970da4e00050cd28a202fb53a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 2332/2779] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 025424851..fd6dcefb2 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 11bf3fe84c23f316d212bd93bcde98cbd237bc4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:29 +0100 Subject: [PATCH 2333/2779] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 862e2eccf..e8d331278 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 9180bb752833e6178946378de108889c49fcd9dc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:37:12 +0100 Subject: [PATCH 2334/2779] added pylib and requirements.txt --- .github/workflows/python2.7.yaml | 6 ++++++ .github/workflows/python3.10.yaml | 6 ++++++ .github/workflows/python3.6.yaml | 6 ++++++ .github/workflows/python3.7.yaml | 6 ++++++ .github/workflows/python3.8.yaml | 6 ++++++ .github/workflows/python3.9.yaml | 6 ++++++ 6 files changed, 36 insertions(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 60b31b6bc..ff3e51dd1 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python2.7.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python2.7.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 939ae7337..de6f79440 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.10.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.10.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index d6a61ea45..84dce85e0 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.6.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.6.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 70ca1f828..58ad6c210 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.7.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.7.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index fd6dcefb2..f1c806dc2 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.8.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.8.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index e8d331278..31c7d0ab8 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.9.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.9.yaml workflow_dispatch: inputs: debug: From 2fe17870b260fb815a2db3e63c0e7138b925f6b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 2335/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4bc47ad4e..b352785d0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4bc47ad4e3dc0d23f08054b6f4687a92869ec9f5 +Subproject commit b352785d0250a833a1b77f18f3efbd8d0a9038eb From 1f7dd4b40204da6a26f91bb930fbf3fb36674694 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 2336/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b22457e26..ffe46e95e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b22457e26204158f537524111be4e45669b82663 +Subproject commit ffe46e95e258b70b9e66f1e928291a273b93973b From 1d29f187221e74e7e05c9322a00402b8f9d865a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 2337/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 300fd2a80..ad25bcb23 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 300fd2a80741eaf8dcf3e2900f507a5531355bd7 +Subproject commit ad25bcb23280f7307d8897111a238cc6b688f876 From 2ae16a579a126f1d43d69d9f4748a83bef4ec77a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 2338/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9a3f5c616..de060e1aa 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9a3f5c616667cab76f98c13f2d8576610b877d1f +Subproject commit de060e1aa016757a2411b5ab8e8d35bf4750c9cb From 13675e4f89ec8b98b31c552cca802c7e3199c4c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 2339/2779] updated docker_pytools_alpine.yaml --- .github/workflows/docker_pytools_alpine.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index c3687b351..252e76f7b 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:alpine dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-alpine + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 3204f55973046b07e10b5148d85074e6a0498dbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 2340/2779] updated docker_pytools_centos.yaml --- .github/workflows/docker_pytools_centos.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml index 92b36e7d5..88474229f 100644 --- a/.github/workflows/docker_pytools_centos.yaml +++ b/.github/workflows/docker_pytools_centos.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:centos dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-centos + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 4fb936a61ebc471c3abb1bc378235ce3c343eeaf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 2341/2779] updated docker_pytools_debian.yaml --- .github/workflows/docker_pytools_debian.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index 8b34df315..876ccce1f 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:debian dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-debian + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From d4196934562d74e6a72599ec297fccd8003c1d10 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 2342/2779] updated docker_pytools_fedora.yaml --- .github/workflows/docker_pytools_fedora.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 190cf703b..0299270f3 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:fedora dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-fedora + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 9e62e14fb9aa565b9dc08bb46e8f8814df9d4bfb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:32 +0100 Subject: [PATCH 2343/2779] updated docker_pytools_ubuntu.yaml --- .github/workflows/docker_pytools_ubuntu.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index fe5e51a95..535621a81 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -33,6 +48,7 @@ jobs: ghcr.io/harisekhon/pytools:ubuntu dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-ubuntu + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 898775ec901b4d4cb8ad6204c98c59a69fdd8d1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:01:47 +0100 Subject: [PATCH 2344/2779] added kics.yaml --- .github/workflows/kics.yaml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/kics.yaml diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml new file mode 100644 index 000000000..b73bedf6f --- /dev/null +++ b/.github/workflows/kics.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Kics + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kics: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Kics + uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master From 6216db9880e0ce935a1e264566dad52a911a1e3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:05:30 +0100 Subject: [PATCH 2345/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3aa702615..1561bd333 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Hari Sekhon - DevOps Python Tools [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) +[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) [![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) From f3cc7f678f6a75a4dba04b47ac38c59a5ae28d9a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:31:49 +0100 Subject: [PATCH 2346/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b352785d0..df5a509bb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b352785d0250a833a1b77f18f3efbd8d0a9038eb +Subproject commit df5a509bb8c679914a137c8ff21597863fdd21ac From 131c4a984cb3848411c15b2036216c70ec5dbfbe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:31:49 +0100 Subject: [PATCH 2347/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ffe46e95e..548ebc658 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ffe46e95e258b70b9e66f1e928291a273b93973b +Subproject commit 548ebc658c20f56600cac1bbd87ee1f267274696 From fbed6887015c0207021ef28ce82a4376260a009d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:31:49 +0100 Subject: [PATCH 2348/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index de060e1aa..b81a01bca 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit de060e1aa016757a2411b5ab8e8d35bf4750c9cb +Subproject commit b81a01bca4cca15dd9cf495bb253d1e27693cc7d From d3b9407383090a554a75ec40f452155c4895765b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 May 2022 12:01:24 +0100 Subject: [PATCH 2349/2779] updated URLs --- .appveyor.yml | 4 +- .buildkite/pipeline.yml | 4 +- .circleci/config.yml | 4 +- .cirrus.yml | 4 +- .concourse.yml | 6 +-- .drone.yml | 4 +- .editorconfig | 2 +- .github/ISSUE_TEMPLATE.md | 2 +- .github/workflows/docker_pytools_alpine.yaml | 2 +- .github/workflows/docker_pytools_centos.yaml | 2 +- .github/workflows/docker_pytools_debian.yaml | 2 +- .github/workflows/docker_pytools_fedora.yaml | 2 +- .github/workflows/docker_pytools_ubuntu.yaml | 2 +- .../ghcr_python_ubuntu.yaml.disabled | 2 +- .gitlab-ci.yml | 2 +- .gitmodules | 2 +- .gocd.yml | 6 +-- .semaphore/semaphore.yml | 4 +- .travis.yml | 4 +- .yamllint | 4 +- Jenkinsfile | 6 +-- Makefile | 10 ++--- README.md | 38 +++++++++---------- ambari_ams_metrics.sh | 4 +- ambari_blueprints.py | 4 +- ambari_cancel_all_requests.sh | 4 +- ambari_trigger_service_checks.py | 4 +- anonymize.py | 8 ++-- anonymize_custom.conf | 8 +++- anonymize_ignore.conf | 2 +- anonymize_parallel.sh | 4 +- aws_s3_presign.py | 4 +- aws_users_access_key_age.py | 8 ++-- aws_users_last_used.py | 8 ++-- aws_users_pw_last_used.py | 8 ++-- aws_users_unused_access_keys.py | 8 ++-- azure-pipelines.yml | 2 +- bitbucket-pipelines.yml | 4 +- buddy.yml | 6 +-- buildspec.yml | 2 +- center.py | 4 +- cloudbuild.yaml | 2 +- cloudera_navigator_tables_used.py | 4 +- cloudera_navigator_tables_used_postgres.py | 4 +- codefresh.yml | 4 +- codeship.yml | 2 +- crunch_accounting_csv_statement_converter.py | 4 +- ...unch_accounting_csv_statement_converter.sh | 2 +- ...unch_accounting_csv_statement_converter.sh | 2 +- .../lib.sh | 2 +- docker_pull_all_images.sh | 4 +- docker_pull_all_images_all_tags.sh | 4 +- docker_pull_all_tags.sh | 4 +- docker_registry_show_tags.py | 4 +- dockerfiles_check_git_branches.py | 4 +- dockerfiles_check_git_tags.py | 4 +- dockerhub_search.py | 6 +-- dockerhub_show_tags.py | 4 +- find_active_apache_drill.py | 4 +- find_active_apache_drill2.py | 4 +- find_active_cassandra.py | 4 +- find_active_elasticsearch.py | 4 +- find_active_hadoop2_namenode.py | 4 +- find_active_hadoop_namenode.py | 4 +- find_active_hadoop_yarn_resource_manager.py | 4 +- find_active_hbase_master.py | 4 +- find_active_hbase_stargate.py | 4 +- find_active_hbase_thrift.py | 4 +- find_active_impala.py | 4 +- find_active_impala_catalog.py | 4 +- find_active_impala_statestore.py | 4 +- find_active_kubernetes_api.py | 4 +- find_active_oozie.py | 4 +- find_active_presto_coordinator.py | 4 +- find_active_server.py | 6 +-- find_active_solrcloud.py | 4 +- find_duplicate_files.py | 4 +- find_missing_files_in_sequence.py | 2 +- getent.py | 4 +- git_check_branches_upstream.py | 6 +-- hbase_compact_tables.py | 4 +- hbase_flush_tables.py | 4 +- hbase_generate_data.py | 4 +- hbase_region_requests.py | 4 +- hbase_regions_by_memstore_size.py | 4 +- hbase_regions_by_size.py | 4 +- hbase_regions_least_used.py | 4 +- hbase_regionserver_requests.py | 4 +- hbase_scan_table_column_names.sh | 4 +- hbase_show_table_region_ranges.py | 4 +- hbase_table_region_row_distribution.py | 4 +- hbase_table_regions_by_regionserver.sh | 4 +- hbase_table_regionserver_distribution.sh | 4 +- hbase_table_row_key_distribution.py | 4 +- hbase_uncompacted_regions.sh | 4 +- hdfs_files_native_checksums.jy | 4 +- hdfs_files_stats.jy | 4 +- hdfs_find_replication_factor_1.py | 4 +- hdfs_time_block_reads.jy | 4 +- headtail.py | 4 +- hexanonymize.py | 2 +- hive_compute_table_stats.py | 4 +- hive_foreach_table.py | 4 +- hive_schemas_csv.py | 4 +- hive_tables_column_counts.py | 4 +- hive_tables_list.py | 4 +- hive_tables_locations.py | 4 +- hive_tables_metadata.py | 4 +- hive_tables_null_columns.py | 4 +- hive_tables_null_rows.py | 4 +- hive_tables_row_column_counts.py | 4 +- hive_tables_row_counts.py | 4 +- hive_tables_row_counts_any_nulls.py | 4 +- impala_compute_table_stats.py | 4 +- impala_foreach_table.py | 4 +- impala_schemas_csv.py | 4 +- impala_tables_column_counts.py | 4 +- impala_tables_list.py | 4 +- impala_tables_locations.py | 4 +- impala_tables_metadata.py | 4 +- impala_tables_null_columns.py | 4 +- impala_tables_null_rows.py | 4 +- impala_tables_row_column_counts.py | 4 +- impala_tables_row_counts.py | 4 +- impala_tables_row_counts_any_nulls.py | 4 +- ipython_notebook_pyspark.py | 4 +- json_docs_to_bulk_multiline.py | 4 +- json_to_xml.py | 2 +- json_to_yaml.py | 2 +- jython_autoinstall.exp | 4 +- jython_install.sh | 4 +- lib/hive_impala_cli.py | 4 +- lib/postgres_cli.py | 4 +- opentsdb_import_metric_distribution.py | 4 +- opentsdb_list_metrics.sh | 4 +- opentsdb_list_metrics_hbase.sh | 4 +- pig_udfs.jy | 2 +- python_find_library_path.py | 4 +- pythonpath.py | 4 +- quay_show_tags.py | 4 +- serf_event_handler.py | 4 +- setup/apk-packages-dev.txt | 4 +- setup/apk-packages-pip.txt | 4 +- setup/apk-packages.txt | 4 +- setup/bootstrap.sh | 6 +-- setup/brew-packages.txt | 4 +- setup/ci_bootstrap.sh | 4 +- setup/deb-packages-dev.txt | 4 +- setup/deb-packages-optional.txt | 4 +- setup/deb-packages-pip.txt | 4 +- setup/deb-packages.txt | 4 +- setup/gocd_config_repo.json | 2 +- setup/jenkins-job.xml | 4 +- setup/rpm-packages-dev.txt | 4 +- setup/rpm-packages-optional.txt | 4 +- setup/rpm-packages-pip.txt | 4 +- setup/rpm-packages.txt | 4 +- shippable.yml | 4 +- sonar-project.properties | 10 ++--- spark_avro_to_parquet.py | 4 +- spark_csv_to_avro.py | 4 +- spark_csv_to_parquet.py | 4 +- spark_json_to_avro.py | 4 +- spark_json_to_parquet.py | 4 +- spark_parquet_to_avro.py | 4 +- strip_ansi_escape_codes.py | 4 +- tests/all.sh | 4 +- tests/check.sh | 4 +- tests/compile.sh | 4 +- tests/data/add_ou.ldif | 4 +- tests/data/ldap_download.sh | 2 +- tests/data/ldap_upload.sh | 2 +- tests/docker/apache-drill-docker-compose.yml | 4 +- tests/docker/common.yml | 4 +- tests/docker/elasticsearch-docker-compose.yml | 4 +- ...lasticsearch-elastic.co-docker-compose.yml | 4 +- tests/docker/hadoop-docker-compose.yml | 4 +- tests/docker/hbase-docker-compose.yml | 4 +- tests/docker/presto-dev-docker-compose.yml | 4 +- tests/docker/presto-docker-compose.yml | 4 +- tests/docker/registry-docker-compose.yml | 4 +- tests/docker/solrcloud-docker-compose.yml | 4 +- tests/excluded.sh | 4 +- tests/help.sh | 4 +- tests/python3.sh | 4 +- tests/syntax.sh | 4 +- tests/test_anonymize.sh | 4 +- tests/test_apache-drill.sh | 4 +- tests/test_center.sh | 2 +- tests/test_docker.sh | 4 +- tests/test_docker_registry_show_tags.sh | 4 +- tests/test_dockerfiles_check_git_branches.sh | 6 +-- tests/test_dockerfiles_check_git_tags.sh | 6 +-- tests/test_dockerhub_search.sh | 4 +- tests/test_dockerhub_show_tags.sh | 4 +- tests/test_elasticsearch.sh | 4 +- tests/test_find_active_server.sh | 2 +- tests/test_find_duplicate_files.sh | 4 +- tests/test_find_python_library_path.sh | 4 +- tests/test_getent.sh | 4 +- tests/test_git_check_branches_upstream.sh | 6 +-- tests/test_hadoop.sh | 4 +- tests/test_hbase.sh | 4 +- tests/test_headtail.sh | 4 +- tests/test_json.sh | 4 +- tests/test_json_docs_to_bulk_multiline.sh | 4 +- tests/test_json_to_xml.sh | 4 +- tests/test_json_to_yaml.sh | 4 +- tests/test_opentsdb.sh | 4 +- tests/test_presto.sh | 4 +- tests/test_quay_show_tags.sh | 4 +- tests/test_serf_event_handler.sh | 4 +- tests/test_solrcloud.sh | 4 +- tests/test_spark_csv_to_avro.sh | 4 +- tests/test_spark_csv_to_parquet.sh | 4 +- tests/test_spark_json_to_avro.sh | 4 +- tests/test_spark_json_to_parquet.sh | 4 +- tests/test_spark_z_avro_to_parquet.sh | 4 +- tests/test_spark_z_parquet_to_avro.sh | 4 +- tests/test_timeout.sh | 2 +- tests/test_validate_avro.sh | 4 +- tests/test_validate_cson.sh | 4 +- tests/test_validate_csv.sh | 4 +- tests/test_validate_ini.sh | 4 +- tests/test_validate_json.sh | 4 +- tests/test_validate_ldap_ldif.sh | 4 +- tests/test_validate_multimedia.sh | 4 +- tests/test_validate_parquet.sh | 4 +- tests/test_validate_toml.sh | 4 +- tests/test_validate_xml.sh | 4 +- tests/test_validate_yaml.sh | 4 +- tests/test_welcome.sh | 4 +- tests/test_xml_to_json.sh | 4 +- tests/test_xml_to_yaml.sh | 4 +- tests/test_yamllint.sh | 4 +- tests/utils.sh | 4 +- timeout.py | 4 +- travis_debug_session.py | 4 +- travis_last_log.py | 4 +- urldecode.py | 4 +- urlencode.py | 4 +- validate_avro.py | 4 +- validate_cson.py | 4 +- validate_csv.py | 4 +- validate_ini.py | 4 +- validate_ini2.py | 4 +- validate_json.py | 4 +- validate_ldap_ldif.py | 4 +- validate_multimedia.py | 4 +- validate_parquet.py | 4 +- validate_toml.py | 4 +- validate_xml.py | 4 +- validate_yaml.py | 4 +- welcome.py | 4 +- wercker.yml | 4 +- xml_to_json.py | 2 +- xml_to_yaml.py | 2 +- yaml_to_json.py | 2 +- 258 files changed, 532 insertions(+), 528 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 1551a3473..b706d6f72 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://www.appveyor.com/docs/appveyor-yml/ diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 53dd7405b..93f047c02 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # BuildKite Pipeline diff --git a/.circleci/config.yml b/.circleci/config.yml index 37332c73c..5dbde36b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,13 +5,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Master Template with more advanced config: diff --git a/.cirrus.yml b/.cirrus.yml index 7f1425a4c..d544aeb85 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://cirrus-ci.org/guide/writing-tasks/ diff --git a/.concourse.yml b/.concourse.yml index 9ed7da3e2..42b806f26 100644 --- a/.concourse.yml +++ b/.concourse.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # resources: @@ -18,7 +18,7 @@ resources: icon: github-circle type: git source: - uri: https://github.com/harisekhon/devops-python-tools + uri: https://github.com/HariSekhon/DevOps-Python-tools branch: master #- name: daily # type: time diff --git a/.drone.yml b/.drone.yml index 790bdcf4c..4d36c9db2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6,13 +6,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://docs.drone.io/quickstart/cli/ diff --git a/.editorconfig b/.editorconfig index 268166e72..aac36b310 100644 --- a/.editorconfig +++ b/.editorconfig @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 5f04c0a58..aa46b9847 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,3 +1,3 @@ Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell. -You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively. +You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) respectively. diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index 252e76f7b..8ce10e111 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml index 88474229f..e8db3ff9c 100644 --- a/.github/workflows/docker_pytools_centos.yaml +++ b/.github/workflows/docker_pytools_centos.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index 876ccce1f..2b92c8282 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 0299270f3..92a93a6b4 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index 535621a81..d6a4ddcb0 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ghcr_python_ubuntu.yaml.disabled b/.github/workflows/ghcr_python_ubuntu.yaml.disabled index fdd5ad61b..c4503bc13 100644 --- a/.github/workflows/ghcr_python_ubuntu.yaml.disabled +++ b/.github/workflows/ghcr_python_ubuntu.yaml.disabled @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af2a6a22b..f9a691721 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: Sun Feb 23 19:02:10 2020 +0000 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # diff --git a/.gitmodules b/.gitmodules index 2bb07ff73..455bce3c9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "pylib"] path = pylib - url = https://github.com/harisekhon/pylib + url = https://github.com/HariSekhon/pylib branch = master [submodule "bash-tools"] path = bash-tools diff --git a/.gocd.yml b/.gocd.yml index 69d289bf4..d3a89a0ef 100644 --- a/.gocd.yml +++ b/.gocd.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://github.com/tomzo/gocd-yaml-config-plugin#setup @@ -26,7 +26,7 @@ pipelines: display_order: -1 materials: git: - git: https://github.com/harisekhon/devops-python-tools + git: https://github.com/HariSekhon/DevOps-Python-tools shallow_clone: false auto_update: true branch: master diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 99040a203..dd256f26c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ diff --git a/.travis.yml b/.travis.yml index 71348b0bc..47f2f80b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,14 @@ # Author: Hari Sekhon # Date: 2014-11-29 01:02:47 +0000 (Sat, 29 Nov 2014) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/.yamllint b/.yamllint index b5e6f8448..e29153b7f 100644 --- a/.yamllint +++ b/.yamllint @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/Jenkinsfile b/Jenkinsfile index 7739f92e3..c4ff34fb2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,13 +4,13 @@ // Author: Hari Sekhon // Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017) // -// https://github.com/harisekhon/devops-python-tools +// https://github.com/HariSekhon/DevOps-Python-tools // // License: see accompanying Hari Sekhon LICENSE file // // If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish // -// https://www.linkedin.com/in/harisekhon +// https://www.linkedin.com/in/HariSekhon // // ========================================================================== // @@ -49,7 +49,7 @@ pipeline { stages { stage ('Checkout') { steps { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/HariSekhon/DevOps-Python-tools']]]) } } diff --git a/Makefile b/Makefile index b7f961b67..ae61cbc5e 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules to non-system python otherwise they're not found by programs. @@ -21,15 +21,15 @@ # # Alpine: # -# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make +# apk add --no-cache git make && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # # Debian / Ubuntu: # -# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make +# apt-get update && apt-get install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # # RHEL / CentOS: # -# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make +# yum install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # =================== diff --git a/README.md b/README.md index 1561bd333..535e5389a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ Hari Sekhon - DevOps Python Tools ================================= -[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/stargazers) -[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/network) +[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) @@ -116,10 +116,10 @@ DevOps, Cloud, Big Data, NoSQL, Python & Linux tools. All programs have `--help` See Also: - - [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - - [DevOps Perl Tools](https://github.com/harisekhon/devops-perl-tools) + - [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) + - [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) - [DevOps Golang Tools](https://github.com/HariSekhon/DevOps-Golang-tools) - - [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) + - [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) repos which contains hundreds more scripts and programs for Cloud, Big Data, SQL, NoSQL, Web and Linux. @@ -127,7 +127,7 @@ Hari Sekhon Cloud & Big Data Contractor, United Kingdom -[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/harisekhon/) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/HariSekhon/) ###### (you're welcome to connect with me on LinkedIn) ##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### @@ -156,7 +156,7 @@ curl -L https://git.io/python-bootstrap | sh or manually: ``` -git clone https://github.com/harisekhon/devops-python-tools pytools +git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools make ``` @@ -166,9 +166,9 @@ To only install pip dependencies for a single script, you can just type make and make anonymize.pyc ``` -Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/devops-python-tools#detailed-build-instructions) further down for more information. +Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/DevOps-Python-tools#detailed-build-instructions) further down for more information. -Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/harisekhon/devops-python-tools#jython-for-hadoop-utils) for details. +Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/HariSekhon/DevOps-Python-tools#jython-for-hadoop-utils) for details. ### Usage @@ -290,7 +290,7 @@ Environment variables are supported for convenience and also to hide credentials - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) + - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI @@ -321,7 +321,7 @@ The automated build will use 'sudo' to install required Python PyPI libraries to Enter the pytools directory and run git submodule init and git submodule update to fetch my library repo: ``` -git clone https://github.com/harisekhon/devops-python-tools pytools +git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools git submodule init git submodule update @@ -333,7 +333,7 @@ sudo pip install -r requirements.txt Download the DevOps Python Tools and Pylib git repos as zip files: -https://github.com/HariSekhon/devops-python-tools/archive/master.zip +https://github.com/HariSekhon/DevOps-Python-tools/archive/master.zip https://github.com/HariSekhon/pylib/archive/master.zip @@ -433,7 +433,7 @@ Then add the Jython install bin directory to the $PATH or specify the full path #### Configuration for Strict Domain / FQDN validation #### -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/harisekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). #### Python SSL certificate verification problems @@ -457,9 +457,9 @@ If you update often and want to just quickly git pull + submodule update but ski ### Testing [Continuous Integration](https://travis-ci.org/HariSekhon/devops-python-tools) is run on this repo with tests for success and failure scenarios: -- unit tests for the custom supporting [python library](https://github.com/harisekhon/pylib) +- unit tests for the custom supporting [python library](https://github.com/HariSekhon/pylib) - integration tests of the top level programs using the libraries for things like option parsing -- [functional tests](https://github.com/HariSekhon/devops-python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/) +- [functional tests](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/) To trigger all tests run: @@ -475,7 +475,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 700+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 700+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery @@ -483,7 +483,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments -- [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +- [The Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) - [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... @@ -491,9 +491,9 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 -- [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule +- [PyLib](https://github.com/HariSekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule -- [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library +- [Perl Lib](https://github.com/HariSekhon/lib) - Perl version of above library From bccc01b7473cf51d3a5be02a09f379fe3ff569fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:42 +0100 Subject: [PATCH 2487/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0caddf7a5..5e7dfbdc8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0caddf7a58297dd75d19ecfea71eb2206853d4c4 +Subproject commit 5e7dfbdc8cea422b6856153eea3c01df4c84a60c From 5a0a33122adaa05720ea69222186feae21887c6f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:42 +0100 Subject: [PATCH 2488/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 357fcd97c..4515bb958 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 357fcd97c3ccf3b6f89aa07ef71c994504e7c7c8 +Subproject commit 4515bb958f4b27c7c5854ba77ee665b41852dd5b From cf3544f48c0168b9be909b530a3c071d9b46aad7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:43 +0100 Subject: [PATCH 2489/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ff707a116..d4eedc0e2 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ff707a1168889fb3ad759c030f3fc633d23d800f +Subproject commit d4eedc0e2c68c9a43838c89a1d09abf10927a566 From 194ca3daa90416a88ee399a7b9e7c3f0d7bd90c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:43 +0100 Subject: [PATCH 2490/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 7373f570f..8d838daa6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 7373f570ffb1a790ec576ef4c6cf52dcf0785ba0 +Subproject commit 8d838daa669dc05d497d5fb2fa1aef631c613acf From 2b13a79338d74de5b5ea78f59cd02f79484e1c61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:28:26 +0100 Subject: [PATCH 2491/2779] renamed debian_8.yaml to debian_8.yaml.disabled --- .github/workflows/{debian_8.yaml => debian_8.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_8.yaml => debian_8.yaml.disabled} (100%) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml.disabled similarity index 100% rename from .github/workflows/debian_8.yaml rename to .github/workflows/debian_8.yaml.disabled From 8ef52b84f1c5e3aa90439b660ec903ee8c132e05 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:37:40 +0100 Subject: [PATCH 2492/2779] added debian_11.yaml --- .github/workflows/debian_11.yaml | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/debian_11.yaml diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml new file mode 100644 index 000000000..de3b4b96f --- /dev/null +++ b/.github/workflows/debian_11.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 11 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_11.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:11 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} From ed38e3efe36f2d5fc3eae3f90b6f50d82d538a47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:49:50 +0100 Subject: [PATCH 2493/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 152556544..af9216972 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ [![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) -[![Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+8%22) [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) +[![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) [![CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22) [![CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22) [![CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22) From 5ba4480dc89627d3c39d1f8de03f3450d709fa7c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:42:16 +0100 Subject: [PATCH 2494/2779] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index af9216972..c6ae3495f 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,9 @@ [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) -[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml) [![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) - [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) From 824e98dd0d38a3382c69dac5eb7e85b1056836ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:49:30 +0100 Subject: [PATCH 2495/2779] renamed python3.6.yaml to python3.6.yaml.disabled --- .github/workflows/{python3.6.yaml => python3.6.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python3.6.yaml => python3.6.yaml.disabled} (100%) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml.disabled similarity index 100% rename from .github/workflows/python3.6.yaml rename to .github/workflows/python3.6.yaml.disabled From 7408f3ce0898dea20692929e949e336edd98dfe8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:49:36 +0100 Subject: [PATCH 2496/2779] renamed python2.7.yaml to python2.7.yaml.disabled --- .github/workflows/{python2.7.yaml => python2.7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python2.7.yaml => python2.7.yaml.disabled} (100%) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml.disabled similarity index 100% rename from .github/workflows/python2.7.yaml rename to .github/workflows/python2.7.yaml.disabled From 83b8b0506a21b3636ba4f75627f965c01fb0545f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:51:59 +0100 Subject: [PATCH 2497/2779] added python3.11.yaml --- .github/workflows/python3.11.yaml | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/python3.11.yaml diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml new file mode 100644 index 000000000..8e2d041ac --- /dev/null +++ b/.github/workflows/python3.11.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.11 + +on: + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.11 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: "3.11" + caches: apt pip + debug: ${{ github.event.inputs.debug }} From 1cfb67773c00608d7fe6fc109f46a968747f63bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:53:20 +0100 Subject: [PATCH 2498/2779] updated README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index c6ae3495f..5b9392d62 100644 --- a/README.md +++ b/README.md @@ -101,12 +101,11 @@ [![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) [![Python versions](https://img.shields.io/badge/Python-2.7+-3776AB?logo=python&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools) -[![Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+2.7%22) -[![Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.6%22) [![Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22) [![Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22) [![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) +[![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) [![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) [![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) From d824e50952a1bb22fef5061544e37305f6b37623 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:56:31 +0100 Subject: [PATCH 2499/2779] renamed pypy2.yaml to pypy2.yaml.disabled --- .github/workflows/{pypy2.yaml => pypy2.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pypy2.yaml => pypy2.yaml.disabled} (100%) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml.disabled similarity index 100% rename from .github/workflows/pypy2.yaml rename to .github/workflows/pypy2.yaml.disabled From 9091c212bd734a1eb605f29d3ecf516fcb12cbcb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:56:31 +0100 Subject: [PATCH 2500/2779] renamed pypy3.yaml to pypy3.yaml.disabled --- .github/workflows/{pypy3.yaml => pypy3.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pypy3.yaml => pypy3.yaml.disabled} (100%) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml.disabled similarity index 100% rename from .github/workflows/pypy3.yaml rename to .github/workflows/pypy3.yaml.disabled From 8bfd91c7b4649060b5704ac3f092d62ae667ca11 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2501/2779] renamed ubuntu_14.04.yaml to ubuntu_14.04.yaml.disabled --- .../workflows/{ubuntu_14.04.yaml => ubuntu_14.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_14.04.yaml => ubuntu_14.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_14.04.yaml rename to .github/workflows/ubuntu_14.04.yaml.disabled From a4139b161723a350733d1ce22aab019e536a4c80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2502/2779] renamed ubuntu_16.04.yaml to ubuntu_16.04.yaml.disabled --- .../workflows/{ubuntu_16.04.yaml => ubuntu_16.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_16.04.yaml => ubuntu_16.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_16.04.yaml rename to .github/workflows/ubuntu_16.04.yaml.disabled From 46ab2bf3cf7fbb872f451c9d0590a9b2dc8ce803 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2503/2779] renamed ubuntu_18.04.yaml to ubuntu_18.04.yaml.disabled --- .../workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml rename to .github/workflows/ubuntu_18.04.yaml.disabled From d3e2d0be996674fe5ccedc47a9bce75b436532fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2504/2779] renamed ubuntu_20.04.yaml to ubuntu_20.04.yaml.disabled --- .../workflows/{ubuntu_20.04.yaml => ubuntu_20.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_20.04.yaml => ubuntu_20.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_20.04.yaml rename to .github/workflows/ubuntu_20.04.yaml.disabled From 1f744ad1dbf1d66fddae98a6244c7f1f35ece41b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 2505/2779] renamed ubuntu_github.yaml to ubuntu_github.yaml.disabled --- .../workflows/{ubuntu_github.yaml => ubuntu_github.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_github.yaml => ubuntu_github.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_github.yaml rename to .github/workflows/ubuntu_github.yaml.disabled From 72c62d30af56d108aa1554fb4884dd6abe635ba0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:02:48 +0100 Subject: [PATCH 2506/2779] renamed ubuntu_18.04.yaml.disabled to ubuntu_18.04.yaml --- .../workflows/{ubuntu_18.04.yaml.disabled => ubuntu_18.04.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml.disabled => ubuntu_18.04.yaml} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml.disabled rename to .github/workflows/ubuntu_18.04.yaml From 62d24176064c6b4dac09d71938d5f0d6e2b34c58 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:03:08 +0100 Subject: [PATCH 2507/2779] renamed ubuntu_20.04.yaml.disabled to ubuntu_20.04.yaml --- .../workflows/{ubuntu_20.04.yaml.disabled => ubuntu_20.04.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_20.04.yaml.disabled => ubuntu_20.04.yaml} (100%) diff --git a/.github/workflows/ubuntu_20.04.yaml.disabled b/.github/workflows/ubuntu_20.04.yaml similarity index 100% rename from .github/workflows/ubuntu_20.04.yaml.disabled rename to .github/workflows/ubuntu_20.04.yaml From 209f3bfd5641f436862ffca3dece34d0bf81b3fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:05:48 +0100 Subject: [PATCH 2508/2779] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5b9392d62..32ee3ce53 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,6 @@ [![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) -[![Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+14.04%22) -[![Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+16.04%22) [![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) From e550318685f87e7e2c25b400e634175915a8debe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:06:12 +0100 Subject: [PATCH 2509/2779] renamed ubuntu_github.yaml.disabled to ubuntu_github.yaml --- .../workflows/{ubuntu_github.yaml.disabled => ubuntu_github.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_github.yaml.disabled => ubuntu_github.yaml} (100%) diff --git a/.github/workflows/ubuntu_github.yaml.disabled b/.github/workflows/ubuntu_github.yaml similarity index 100% rename from .github/workflows/ubuntu_github.yaml.disabled rename to .github/workflows/ubuntu_github.yaml From c0e1bb251c0c9028b9f5b0907788e7766f346acf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:07:38 +0100 Subject: [PATCH 2510/2779] renamed ubuntu_18.04.yaml to ubuntu_18.04.yaml.disabled --- .../workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml rename to .github/workflows/ubuntu_18.04.yaml.disabled From cbf21a97aabc95628d71f773f08229cab9ac8614 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:10:29 +0100 Subject: [PATCH 2511/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32ee3ce53..138775975 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,8 @@ [![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) -[![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) +[![Ubuntu 22.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2022.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+22.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) From a5ad66e4090134ecc51d4dbb869251e63f8f27c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:40 +0100 Subject: [PATCH 2512/2779] renamed centos.yaml to centos.yaml.disabled --- .github/workflows/{centos.yaml => centos.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos.yaml => centos.yaml.disabled} (100%) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml.disabled similarity index 100% rename from .github/workflows/centos.yaml rename to .github/workflows/centos.yaml.disabled From af02db5511f56a40b40a90702619c5dfd8f600a3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:40 +0100 Subject: [PATCH 2513/2779] renamed centos7.yaml to centos7.yaml.disabled --- .github/workflows/{centos7.yaml => centos7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos7.yaml => centos7.yaml.disabled} (100%) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml.disabled similarity index 100% rename from .github/workflows/centos7.yaml rename to .github/workflows/centos7.yaml.disabled From 5b91ee48587426db040ac1afb31fbfef8f1fbeff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:41 +0100 Subject: [PATCH 2514/2779] renamed centos8.yaml to centos8.yaml.disabled --- .github/workflows/{centos8.yaml => centos8.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos8.yaml => centos8.yaml.disabled} (100%) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml.disabled similarity index 100% rename from .github/workflows/centos8.yaml rename to .github/workflows/centos8.yaml.disabled From 79fa99d693ea366638e1ee6c38bd5dec65e4ee43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:28:28 +0100 Subject: [PATCH 2515/2779] updated README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 138775975..72b1c3aab 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,6 @@ [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) [![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) -[![CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22) -[![CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22) -[![CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22) [![Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Fedora%22) [![Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine%22) [![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) From 78ba04ce9d93aedfee00e09cb99c083884a53e1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:48:44 +0100 Subject: [PATCH 2516/2779] updated serf_event_handler.py --- serf_event_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serf_event_handler.py b/serf_event_handler.py index 455d7bee1..c0d5cacbb 100755 --- a/serf_event_handler.py +++ b/serf_event_handler.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.2' +__version__ = '0.2.3' class SerfEventHandler(CLI): @@ -95,7 +95,7 @@ def add_options(self): def enable_commands(self): if self.event in ['query', 'event']: - cmd = None + cmd = '' if self.event == 'query': cmd = self.query_name elif self.event == 'user': From f66248cc4545fe236327911857290247eba1a701 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:49:41 +0100 Subject: [PATCH 2517/2779] updated spark_json_to_avro.py --- spark_json_to_avro.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index 2922c7f7f..d53a4bffa 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -109,7 +109,6 @@ def run(self): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): df = sqlContext.read.json(json_file) else: From 8446d976d4efcee77aad458c5086d16c86a74255 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:53:59 +0100 Subject: [PATCH 2518/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 72b1c3aab..e12942015 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools) [![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) From d80f92c71f9aefed3a161dc9cf719766ad4c1eef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:05:15 +0100 Subject: [PATCH 2519/2779] renamed docker_pytools_centos.yaml to *_centos.yaml.disabled --- .../{docker_pytools_centos.yaml => *_centos.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{docker_pytools_centos.yaml => *_centos.yaml.disabled} (100%) diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/*_centos.yaml.disabled similarity index 100% rename from .github/workflows/docker_pytools_centos.yaml rename to .github/workflows/*_centos.yaml.disabled From 930a2c341a40292160d34cfc827a6b8358f306a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:06:10 +0100 Subject: [PATCH 2520/2779] updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index e12942015..c301b354d 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,6 @@ [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) -[![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) [![Docker Build (Fedora)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml) [![Docker Build (Ubuntu)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml) From 2f629017ce6a08b0db62bff40c1a90f5f801ab53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:11:11 +0100 Subject: [PATCH 2521/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 5e7dfbdc8..bccee5cd7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 5e7dfbdc8cea422b6856153eea3c01df4c84a60c +Subproject commit bccee5cd73f9bb86ab84609681defcea0ec23b9d From a109075a96e1d21f3dbe6cc21dd76b4a0706359e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:11:11 +0100 Subject: [PATCH 2522/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4515bb958..47048ee0c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4515bb958f4b27c7c5854ba77ee665b41852dd5b +Subproject commit 47048ee0c39d1e88a10d09fec64333b2de8e906b From 5bdc0c0db940cbc74019b7b37508c6943e0b65f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:33:03 +0100 Subject: [PATCH 2523/2779] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index c301b354d..edb49e6b1 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,6 @@ [![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) [![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) -[![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) -[![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) [git.io/pytools](https://git.io/pytools) From e2f37bf682f28263a993209c5e52bcef272025d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:36:37 +0100 Subject: [PATCH 2524/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bccee5cd7..995ad2de8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bccee5cd73f9bb86ab84609681defcea0ec23b9d +Subproject commit 995ad2de8486a96487e68c859c4f353a381b91d7 From 8bb7bc784d8f0ad98dbefd691ee8bc460f727330 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:36:37 +0100 Subject: [PATCH 2525/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 47048ee0c..7c0ba3afe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 47048ee0c39d1e88a10d09fec64333b2de8e906b +Subproject commit 7c0ba3afe47f47af186cadab75b508a69d4411fe From 383114f4538c6ccc9f2443d5c53d96e6f2c3d526 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:39:36 +0100 Subject: [PATCH 2526/2779] added ubuntu_22.04.yaml --- .github/workflows/ubuntu_22.04.yaml | 85 +++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/ubuntu_22.04.yaml diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml new file mode 100644 index 000000000..b9bf17a95 --- /dev/null +++ b/.github/workflows/ubuntu_22.04.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 22.04 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_22.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:22.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} From 2769fba289020e9b94311b1f1bee3bcefc02f60e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:43:07 +0100 Subject: [PATCH 2527/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 995ad2de8..fe25b15b0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 995ad2de8486a96487e68c859c4f353a381b91d7 +Subproject commit fe25b15b096e5e667b0451ebc9f68cc86c87384f From 6bfe3eb1a41357c469d3895b6739dfea28d7fab3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:43:07 +0100 Subject: [PATCH 2528/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7c0ba3afe..0a225da5c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7c0ba3afe47f47af186cadab75b508a69d4411fe +Subproject commit 0a225da5c8aa09653d53dd1aaad545d200ec7e7a From 55827d1746db9186a49880e91d4f6d711058e7fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 18:50:55 +0100 Subject: [PATCH 2529/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fe25b15b0..2295778d8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fe25b15b096e5e667b0451ebc9f68cc86c87384f +Subproject commit 2295778d8039333abc3e159754641b348cf79d94 From 5491f93ea4295dd3a87c7541a5182b146baadfe7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 18:50:56 +0100 Subject: [PATCH 2530/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0a225da5c..6e5e84d30 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0a225da5c8aa09653d53dd1aaad545d200ec7e7a +Subproject commit 6e5e84d301329d0386b8a3468e4f7b34700d3e32 From a9c81105474dbfad815563507e21fa7b6f5d8785 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 19:07:39 +0100 Subject: [PATCH 2531/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2295778d8..37dede02a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2295778d8039333abc3e159754641b348cf79d94 +Subproject commit 37dede02a5a184087798740bfbb36b90cf5ed685 From 3680e4c7ab65c6ef873bdf8151fe86efe68fd16a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 19:07:39 +0100 Subject: [PATCH 2532/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6e5e84d30..5f2f181a9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6e5e84d301329d0386b8a3468e4f7b34700d3e32 +Subproject commit 5f2f181a96f884acd0340125aab776414c343b68 From 8865ba8731144538dd0baba23d1ccb787c5988a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 May 2023 17:50:27 +0100 Subject: [PATCH 2533/2779] added --unspace functionality --- center.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/center.py b/center.py index 4856ea0f7..b9cbac5f1 100755 --- a/center.py +++ b/center.py @@ -44,7 +44,8 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.2' +__version__ = '0.5.0' + class Center(CLI): @@ -56,7 +57,9 @@ def __init__(self): # this doesn't put enough spaces around ampersands, eg. in "Auth & Config" #self.re_bound = re.compile(r'(\b)') self.re_spaces = re.compile(r'(\s)') + self.re_multiple_spaces = re.compile(r'(\s){2}') self.re_chars = re.compile(r'([^\s])(?!\s)') + self.re_chars_spaced = re.compile(r'([^\s])\s') self.timeout_default = None def add_options(self): @@ -66,11 +69,16 @@ def add_options(self): help='No comment prefix handling') self.add_opt('-s', '--space', action='store_true', default=False, help='Space all chars out, makes bigger headings') + self.add_opt('-u', '--unspace', action='store_true', default=False, + help='Removes spaces betweeen chars out, the inverse of --space') def run(self): log_option('width', self.get_opt('width')) log_option('no comment prefix', self.get_opt('no_comment')) log_option('space chars', self.get_opt('space')) + log_option('unspace chars', self.get_opt('unspace')) + if self.get_opt('space') and self.get_opt('unspace'): + self.usage("--space and --unspace are mutually exclusive!") if self.args: self.process_line(' '.join(self.args)) else: @@ -83,6 +91,11 @@ def space(self, line): line = self.re_chars.sub(r'\1 ', line) return line + def unspace(self, line): + line = self.re_chars_spaced.sub(r'\1', line) + line = self.re_multiple_spaces.sub(r'\1', line) + return line + def process_line(self, line): char = '' if not line: @@ -102,9 +115,12 @@ def process_line(self, line): line = line.lstrip(char) if self.get_opt('space'): line = self.space(line) + if self.get_opt('unspace'): + line = self.unspace(line) line = line.strip() side = int(max((self.get_opt('width') - len(line)) / 2, 0)) print(char + ' ' * side + line) + if __name__ == '__main__': Center().main() From ad1c2e05e9ebe50dcc9829209328c2fa9c3b9016 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 May 2023 17:50:49 +0100 Subject: [PATCH 2534/2779] updated .pylintrc --- .pylintrc | 751 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 508 insertions(+), 243 deletions(-) diff --git a/.pylintrc b/.pylintrc index c20ac8f94..1c56d5de1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,386 +1,651 @@ -[MASTER] +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y L i n t C o n f i g +# ============================================================================ # + +# pylint --generate-rcfile >> .pylintrc + +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= -# Specify a configuration file. -#rcfile= +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 -# Pickle collected data for later comparisons. -persistent=yes +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 -# List of plugins (as comma separated values of python modules names) to load, +# List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= -# Use multiple processes to speed up Pylint. -jobs=1 +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. -optimize-ast=no +[BASIC] -[MESSAGES CONTROL] +# Naming style matching correct argument names. +argument-naming-style=snake_case -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= +# Naming style matching correct attribute names. +attr-naming-style=snake_case -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111,consider-using-f-string +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata -[REPORTS] +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text +# Naming style matching correct class attribute names. +class-attribute-naming-style=any -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= -# Tells whether to display a full report or only the messages -reports=yes +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= +# Naming style matching correct class names. +class-naming-style=PascalCase +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= -[BASIC] +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,input +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +# Naming style matching correct method names. +method-naming-style=snake_case -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ -# Regular expression matching correct function names -function-rgx=[A-Za-z_][A-Za-z0-9_]{2,30}$ +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Naming style matching correct variable names. +variable-naming-style=snake_case -# Regular expression matching correct constant names -const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ +[CLASSES] -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ +[DESIGN] -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of arguments for function / method. +max-args=5 -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of attributes for a class (see R0902). +max-attributes=7 -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of branch for function / method body. +max-branches=12 -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ +# Maximum number of locals for function / method body. +max-locals=15 -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 +# Maximum number of return / yield for function / method body. +max-returns=6 -[ELIF] +# Maximum number of statements in function / method body. +max-statements=50 -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] -# Maximum number of characters on a single line. -max-line-length=120 +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator -# Maximum number of lines in a module -max-module-lines=1000 +[IMPORTS] -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= [LOGGING] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + # Logging modules to check that the string format arguments are in logging -# function parameter format +# function parameter format. logging-modules=logging +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO +notes=FIXME, + XXX, + TODO +# Regular expression of note tags to take in consideration. +notes-rgx= -[SIMILARITIES] -# Minimum lines number of a similarity. -min-similarity-lines=4 +[REFACTORING] -# Ignore comments when computing similarities. -ignore-comments=yes +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 -# Ignore docstrings when computing similarities. -ignore-docstrings=yes +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error -# Ignore imports when computing similarities. -ignore-imports=no +[REPORTS] -[SPELLING] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= -# List of comma separated words that should not be checked. -spelling-ignore-words= +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= +# Tells whether to display a full report or only the messages. +reports=no -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no +# Activate the evaluation score. +score=yes -[TYPECHECK] +[SIMILARITIES] -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes +# Comments are removed from the similarity computation +ignore-comments=yes -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= +# Docstrings are removed from the similarity computation +ignore-docstrings=yes -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). This supports can work -# with qualified names. -# -# avoids the following error: -# -# pylint -E ./check_zookeeper_version.py -# -# E: 69,12: Instance of '_socketobject' has no 'sendall' member (no-member) -# E: 70,19: Instance of '_socketobject' has no 'recv' member (no-member) -# -ignored-classes=SQLObject,_socketobject +# Imports are removed from the similarity computation +ignore-imports=yes -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= +# Signatures are removed from the similarity computation +ignore-signatures=yes +# Minimum lines number of a similarity. +min-similarity-lines=4 -[VARIABLES] -# Tells whether we should check for unused import in __init__ files. -init-import=no +[SPELLING] -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: +# List of comma separated words that should not be checked. +spelling-ignore-words= -[CLASSES] +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs +[STRING] -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no -[DESIGN] -# Maximum number of arguments for function / method -max-args=5 +[TYPECHECK] -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager -# Maximum number of locals for function / method body -max-locals=15 +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= -# Maximum number of return / yield for function / method body -max-returns=6 +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes -# Maximum number of branch for function / method body -max-branches=12 +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes -# Maximum number of statements in function / method body -max-statements=50 +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init -# Maximum number of parents for a class (see R0901). -max-parents=7 +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace -# Maximum number of attributes for a class (see R0902). -max-attributes=7 +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin +# List of decorators that change the signature of a decorated function. +signature-mutators= -[IMPORTS] -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec +[VARIABLES] -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= +# List of names allowed to shadow builtins +allowed-redefined-builtins= +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb -[EXCEPTIONS] +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io From 64b326acbcea84535f033b706e6bfb5b289a5f32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 03:29:52 +0100 Subject: [PATCH 2535/2779] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index edb49e6b1..f064e78b0 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ [![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) -[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 0487abce888a1fdd84a18f853ef1871e17772147 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 2536/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 37dede02a..09a50e7e1 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 37dede02a5a184087798740bfbb36b90cf5ed685 +Subproject commit 09a50e7e1810c856eff735279a83c00116a7fcd5 From 493c6d1afb562ca7eca054bde3d48fd90b9586c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 2537/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5f2f181a9..c3ab2659f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5f2f181a96f884acd0340125aab776414c343b68 +Subproject commit c3ab2659f369a3af7ee16344ef52e222fcd7a555 From a0fed602600e01b0aa97e35e6824560e71de1bb9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 2538/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 8d838daa6..0d8bd4872 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 8d838daa669dc05d497d5fb2fa1aef631c613acf +Subproject commit 0d8bd48723672bbfec6eb549c80e564b28725a7b From dd64db0434d294fc3bd53a2858e4da7920667009 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:04:10 +0100 Subject: [PATCH 2539/2779] added kics.config --- kics.config | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 kics.config diff --git a/kics.config b/kics.config new file mode 100644 index 000000000..5fafd16d2 --- /dev/null +++ b/kics.config @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-05 18:05:53 +0100 (Fri, 05 May 2023) +# +# vim:ts=2:sts=2:sw=2:et:filetype=yaml +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# K i c s C o n f i g +# ============================================================================ # + +# https://github.com/Checkmarx/kics/blob/master/docs/configuration-file.md + +--- +#path: assets/iac_samples +verbose: true +log-file: true +#type: +# - Dockerfile +# - Kubernetes +#queries-path: "assets/queries" +exclude-paths: + # ignore submodules - handle them in the source repos only + - bash-tools/ + - pylib/ + - sql/ + - templates/ +#output-path: "results" From fee21a0c50c83186949d1ae3253f0882ae642054 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 2540/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 09a50e7e1..b2bcc4c93 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 09a50e7e1810c856eff735279a83c00116a7fcd5 +Subproject commit b2bcc4c93fcfc3e89421d8d264e6a863d623cf32 From 131da36d872ec1e3b2013de15589703b594f5323 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 2541/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c3ab2659f..e137295ce 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c3ab2659f369a3af7ee16344ef52e222fcd7a555 +Subproject commit e137295ce7ae4c484c705a7e800c3577af823cef From 0ba9caa0aa08f5e044c5e340b195ad008944fb1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 2542/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 0d8bd4872..73d5f3b1f 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 0d8bd48723672bbfec6eb549c80e564b28725a7b +Subproject commit 73d5f3b1fa54a3900493417637652f057fab3b61 From 5c929d22aa11b8553ec809a1b11172b933047ec2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 21:54:57 +0100 Subject: [PATCH 2543/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b2bcc4c93..9112b3d95 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b2bcc4c93fcfc3e89421d8d264e6a863d623cf32 +Subproject commit 9112b3d957ce48300a139fc94c1da963c2fa8bf0 From 62046edaf17c3d87f6e7d1f281813e0ec1b2aaf2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 21:54:57 +0100 Subject: [PATCH 2544/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e137295ce..6a6b9cf72 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e137295ce7ae4c484c705a7e800c3577af823cef +Subproject commit 6a6b9cf7211121fa2bb3d8f72dd1ef8ba19bcdf5 From 6f96ebf189863beae3e9612a6deb2743044dee24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 22:51:14 +0100 Subject: [PATCH 2545/2779] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f064e78b0..9c78001bb 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,8 @@ [![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) [![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) From afe9d7a35dc370d604111a9e15c824e4d63f6fe2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:48 +0100 Subject: [PATCH 2546/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9112b3d95..a3750afbd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9112b3d957ce48300a139fc94c1da963c2fa8bf0 +Subproject commit a3750afbd00c67a0c497367be19bd807f05c242b From c9ac6b65bb38bbb6d5fa2375b5810d03fa9e46a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:48 +0100 Subject: [PATCH 2547/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6a6b9cf72..5f2c84abb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6a6b9cf7211121fa2bb3d8f72dd1ef8ba19bcdf5 +Subproject commit 5f2c84abb142dfa218cc29fab89d11ddffa2011f From b2d609d7192f656094daf38192d4a396e5afee3a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:49 +0100 Subject: [PATCH 2548/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 73d5f3b1f..590fd1816 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 73d5f3b1fa54a3900493417637652f057fab3b61 +Subproject commit 590fd18169dfed69dd5861d1b33d9c23b4a71a03 From 0341c77f80096ce8a3c7171d5e92ced882d973e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:25:24 +0100 Subject: [PATCH 2549/2779] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 9c78001bb..2ccdff67a 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,11 @@ [![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) From 472331bac11a4d00a39cb865dcc41540e3cdbc8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:29:06 +0100 Subject: [PATCH 2550/2779] updated .pylintrc --- .pylintrc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 1c56d5de1..b35a5fccc 100644 --- a/.pylintrc +++ b/.pylintrc @@ -448,7 +448,10 @@ disable=raw-checker-failed, suppressed-message, useless-suppression, deprecated-pragma, - use-symbolic-message-instead + use-symbolic-message-instead, + missing-function-docstring, + super-with-arguments, + consider-using-f-string # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 93b2c4f3353810ae7a53f4b6707ec9e1c371b7a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:30:27 +0100 Subject: [PATCH 2551/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5f2c84abb..0f48c7572 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5f2c84abb142dfa218cc29fab89d11ddffa2011f +Subproject commit 0f48c7572c549bb109c3d946c7908048854dec73 From 2c42d10e551497e571f0297b36636f04bfb1025a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:45:22 +0100 Subject: [PATCH 2552/2779] updated .pylintrc --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index b35a5fccc..2006619da 100644 --- a/.pylintrc +++ b/.pylintrc @@ -449,6 +449,7 @@ disable=raw-checker-failed, useless-suppression, deprecated-pragma, use-symbolic-message-instead, + missing-class-docstring, missing-function-docstring, super-with-arguments, consider-using-f-string From 7a42f40f3dfa0d0a02658d82f630a59d3f5d5e8b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:47:54 +0100 Subject: [PATCH 2553/2779] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index fb56e18cf..93ea98595 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -166,7 +166,6 @@ def create_struct(arg): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') @@ -198,5 +197,6 @@ def create_struct(arg): # the databricks avro driver df.write.format('com.databricks.spark.avro').save(avro_dir) + if __name__ == '__main__': SparkCSVToAvro().main() From a282dda7d3eb16d5d8777db108bec344e3d90fff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:43 +0100 Subject: [PATCH 2554/2779] updated spark_avro_to_parquet.py --- spark_avro_to_parquet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py index 9300c4fd1..8528af1a6 100755 --- a/spark_avro_to_parquet.py +++ b/spark_avro_to_parquet.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 246d4df8036306782234ed6c1c4d47a7dee9d44e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:43 +0100 Subject: [PATCH 2555/2779] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index 93ea98595..d41717b4f 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -57,6 +57,7 @@ 'com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From dad1bce7ddc6cc5972ba5df65df8ff0138b1f659 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 2556/2779] updated spark_csv_to_parquet.py --- spark_csv_to_parquet.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py index 8e3b0da1c..e70f014f6 100755 --- a/spark_csv_to_parquet.py +++ b/spark_csv_to_parquet.py @@ -49,6 +49,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-csv_2.11:1.5.0 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error @@ -157,9 +158,7 @@ def create_struct(arg): if not isVersionLax(spark_version): die("Spark version couldn't be determined. " + support_msg('pytools')) - # pylint: disable=invalid-name - - df = None + # pylint: disable=invalid-name if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') From 49c66be7fd0cc50d196ba7e6c1f0a503c14c1771 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 2557/2779] updated spark_json_to_avro.py --- spark_json_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index d53a4bffa..db9d069ed 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 7fb4da436c7016412fe3b7426ab4ea06b676e2e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 2558/2779] updated spark_json_to_parquet.py --- spark_json_to_parquet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py index 2ed935917..6976ce153 100755 --- a/spark_json_to_parquet.py +++ b/spark_json_to_parquet.py @@ -43,6 +43,7 @@ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) sys.exit(4) pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 86727ce5e22f53fa4d84627865c4cad83dddf70e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 2559/2779] updated spark_parquet_to_avro.py --- spark_parquet_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py index 0186981f5..030816963 100755 --- a/spark_parquet_to_avro.py +++ b/spark_parquet_to_avro.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From ec22d9ca18fbd0ec75d1137a2ae11c8001247286 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:50 +0100 Subject: [PATCH 2560/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a3750afbd..1b0b1149f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a3750afbd00c67a0c497367be19bd807f05c242b +Subproject commit 1b0b1149fabbbdb4a1c07cd25d2c1941034eb0a1 From f81d2e16df704a4fee44f20a3a48658b0e1f7c17 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:51 +0100 Subject: [PATCH 2561/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0f48c7572..e53f962a7 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0f48c7572c549bb109c3d946c7908048854dec73 +Subproject commit e53f962a7d0eee7cf1f89ca98552d90c0d6226cb From 1643e01fddfd93d2c301a9a6ca925c1192389ef2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:51 +0100 Subject: [PATCH 2562/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 590fd1816..3a74368ea 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 590fd18169dfed69dd5861d1b33d9c23b4a71a03 +Subproject commit 3a74368ead1544983ac21af7500b380b9fc20b9f From dbc5cec14dea46ef3f851d56988ab9173f8ace9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:01 +0100 Subject: [PATCH 2563/2779] updated Makefile --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index a67a168a9..13e4c9b27 100644 --- a/Makefile +++ b/Makefile @@ -86,12 +86,12 @@ init: # @$(MAKE) $@c %.pyc:: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing - @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ echo Generated $@ %.pyo:: %.py - @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile $< && \ echo && \ echo Generated $@ @@ -118,15 +118,15 @@ python: pylib @# only install pip packages not installed via system packages @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt @#$(SUDO_PIP) $(PIP) install -r requirements.txt - @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt + @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3"; fi - PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3" || : + @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3"; fi + PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3" || : @# for impyla @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || : @@ -154,7 +154,7 @@ python: pylib @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi @# Python >= 2.7 - won't build on 2.6, handle separately and accept failure - @PIP=$(PIP) bash-tools/python_pip_install.sh "ipython[notebook]" || : + @PIP=$(PIP) bash-tools/python/python_pip_install.sh "ipython[notebook]" || : @echo $(MAKE) pycompile @echo @@ -216,7 +216,7 @@ test: .PHONY: basic-test basic-test: test-lib - bash-tools/check_all.sh + bash-tools/checks/check_all.sh .PHONY: install install: build From bf4934fe8873bc1153ac2fef4f796a112fc93483 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:38 +0100 Subject: [PATCH 2564/2779] updated all.sh --- tests/all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/all.sh b/tests/all.sh index 33452eb7c..dbac7b8fb 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -31,7 +31,7 @@ section "Running PyTools ALL" cd "$srcdir/.."; # shellcheck disable=SC1090 # has to be included so that isExcluded function is inherited -. "$srcdir/../bash-tools/check_all.sh" +. "$srcdir/../bash-tools/checks/check_all.sh" #tests/test_yamllint.sh @@ -41,4 +41,4 @@ exit 0 # pyspark not found tests/help.sh -bash-tools/run_tests.sh +bash-tools/checks/run_tests.sh From 9a1f9be8ea238a1bc7cb04291f1eb743308bc09b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:47 +0100 Subject: [PATCH 2565/2779] updated anonymize_parallel.sh --- anonymize_parallel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 1f34f863a..abb7e9d21 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -68,7 +68,7 @@ for filename in $file_list; do echo "Removing any pre-existing parts:" rm -v "$filename".* 2>/dev/null || : echo - "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename" + "$srcdir/bash-tools/bin/split.sh" --parts "$parallelism" "$filename" echo "Anonymizing parts" for file_part in "$filename".*; do cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized" From 7251c68560d2f1f605475a8fb0df98e28c18353f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 11 May 2023 23:02:38 +0100 Subject: [PATCH 2566/2779] updated sonar-project.properties --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 270ec8ac7..7189b1a16 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -21,7 +21,7 @@ sonar.projectDescription=Python / Jython Tools sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues -sonar.links.ci=https://travis-ci.org/HariSekhon/devops-python-tools +sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions sonar.sources=. From 754947957d53ef5ed572bdcbb2b91a6f4f4b583c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 11 May 2023 23:59:10 +0100 Subject: [PATCH 2567/2779] updated README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2ccdff67a..56a02a41e 100644 --- a/README.md +++ b/README.md @@ -469,6 +469,10 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery +- [Jenkins](https://github.com/HariSekhon/Jenkins) - Advanced Jenkinsfile & Jenkins Groovy Shared Library + +- [GitHub-Actions](https://github.com/HariSekhon/GitHub-Actions) - GitHub Actions master template & GitHub Actions Shared Workflows library + - [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. - [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments From 01af37704aed0bcbe5eeb78d0bbcd3e849358b63 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 00:37:00 +0100 Subject: [PATCH 2568/2779] added trivy.yaml --- .github/workflows/trivy.yaml | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/trivy.yaml diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 000000000..5c9475308 --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# T r i v y +# ============================================================================ # + +--- +name: Trivy + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + trivy: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Trivy Filesystem Scan + uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From 0d560dd9f6634ad3ef668f2aa8ee52ab20d0f204 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 00:47:26 +0100 Subject: [PATCH 2569/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 56a02a41e..aeab7f613 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) +[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) -[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) From 585e9050a905151a82746a4a8bfe42b993bf7ed6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 01:08:50 +0100 Subject: [PATCH 2570/2779] updated trivy.yaml --- .github/workflows/trivy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 5c9475308..8af2aeb39 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -1,6 +1,6 @@ # # Author: Hari Sekhon -# Date: Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) # # vim:ts=2:sts=2:sw=2:et # @@ -54,7 +54,7 @@ jobs: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' - name: Trivy Filesystem Scan + name: Trivy uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master with: debug: ${{ github.event.inputs.debug }} From 92b23f826de7fcdab498eb6b9e0ac26b306f09d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 01:41:06 +0100 Subject: [PATCH 2571/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index aeab7f613..a41120672 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) [![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) From 9fe0d3ca28260ec2c4414a6f77c6ebbd0e5c390f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 07:15:53 +0100 Subject: [PATCH 2572/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a41120672..36f322455 100644 --- a/README.md +++ b/README.md @@ -467,7 +467,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ## Related Repositories -- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 800+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 1000+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From b8c21da103ec580fb9b26ade93ad3eeb664cd2db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 14 May 2023 04:18:31 +0100 Subject: [PATCH 2573/2779] added grype.yaml --- .github/workflows/grype.yaml | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/grype.yaml diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml new file mode 100644 index 000000000..6a321f86e --- /dev/null +++ b/.github/workflows/grype.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G r y p e +# ============================================================================ # + +--- +name: Grype + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + Grype: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Grype + uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From a8c29b46dd6d8b0659fb64bbbc41c1d2cd6666e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 May 2023 02:06:58 +0100 Subject: [PATCH 2574/2779] updated center.py --- center.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/center.py b/center.py index b9cbac5f1..379353e68 100755 --- a/center.py +++ b/center.py @@ -44,7 +44,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class Center(CLI): @@ -104,7 +104,7 @@ def process_line(self, line): char = ' ' # preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it line = line.strip() - if isChars(line[0], '#'): + if line and isChars(line[0], '#'): char = line[0] line = line.lstrip(char) elif len(line) > 1 and isChars(line[0:1], '/'): From f9abc5f8321e73b34e0be59079823ce850b50362 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:24 +0100 Subject: [PATCH 2575/2779] removed codeship.yml --- codeship.yml | 58 ---------------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 codeship.yml diff --git a/codeship.yml b/codeship.yml deleted file mode 100644 index f466bd6a7..000000000 --- a/codeship.yml +++ /dev/null @@ -1,58 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2021-04-12 18:33:44 +0100 (Mon, 12 Apr 2021) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# ============================================================================ # -# C o d e S h i p -# ============================================================================ # - -# LEGACY - CloudBees has removed free plans as of 1st March 2023 - -# 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this - -# https://github.com/painless-software/codeship-yaml - -# Requires setting up the CodeShip commands like so: -# -# pip install codeship-yaml -# codeship-yaml -# -# or seaparately in sections: -# -# Project Settings > Test Settings > Setup Commands: -# -# pip install codeship-yaml -# codeship-yaml install -# -# Project Settings > Test Settings > Test Commands: -# -# codeship-yaml before_script script -# -# Project Settings > Deployment > (branch name) -# -# codeship-yaml after_success - ---- -install: - # these cause package installation breakages due to GPG or 403 errors, old addresses etc. - - sudo rm -fv -- /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list - - sudo rm -fv -- /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list - - sudo rm -fv -- /etc/apt/sources.list.d/www_apache_org_dist_cassandra_debian.list - - make -#before_script: -# - somecommand -script: - - make test -#after_success: -# - echo "Now we can deploy" From c4c513b66715f87bbed88320deda71205f84ce96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:32 +0100 Subject: [PATCH 2576/2779] removed wercker.yml --- wercker.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 wercker.yml diff --git a/wercker.yml b/wercker.yml deleted file mode 100644 index c27be30ed..000000000 --- a/wercker.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# https://devcenter.wercker.com/reference/wercker-yml/ - -box: debian - -build: - steps: - - script: - name: ci bootstrap - code: setup/ci_bootstrap.sh - - script: - name: init - code: make init - - script: - name: build - code: make ci - - script: - name: test - code: make test From 116cdfda1b85dbd8803e21a3e126c1d4d58d8ab6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:38 +0100 Subject: [PATCH 2577/2779] removed .werckerignore --- .werckerignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .werckerignore diff --git a/.werckerignore b/.werckerignore deleted file mode 100644 index ff8fb8247..000000000 --- a/.werckerignore +++ /dev/null @@ -1 +0,0 @@ -**/.md From 5ba24dc535c907d2d574d652da6517e3ec2cf75a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:08 +0100 Subject: [PATCH 2578/2779] moved .teamcity.vcs.json to teamcity/ --- .teamcity.vcs.json => teamcity/.teamcity.vcs.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.json => teamcity/.teamcity.vcs.json (100%) diff --git a/.teamcity.vcs.json b/teamcity/.teamcity.vcs.json similarity index 100% rename from .teamcity.vcs.json rename to teamcity/.teamcity.vcs.json From 1730b699555891aff16137749cfbd88f7559f21b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:09 +0100 Subject: [PATCH 2579/2779] moved .teamcity.vcs.oauth.json to teamcity/ --- .teamcity.vcs.oauth.json => teamcity/.teamcity.vcs.oauth.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.oauth.json => teamcity/.teamcity.vcs.oauth.json (100%) diff --git a/.teamcity.vcs.oauth.json b/teamcity/.teamcity.vcs.oauth.json similarity index 100% rename from .teamcity.vcs.oauth.json rename to teamcity/.teamcity.vcs.oauth.json From 21d86fe66caa9b86030bedd0155287ad263ba17f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:09 +0100 Subject: [PATCH 2580/2779] moved .teamcity.vcs.ssh.json to teamcity/ --- .teamcity.vcs.ssh.json => teamcity/.teamcity.vcs.ssh.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.ssh.json => teamcity/.teamcity.vcs.ssh.json (100%) diff --git a/.teamcity.vcs.ssh.json b/teamcity/.teamcity.vcs.ssh.json similarity index 100% rename from .teamcity.vcs.ssh.json rename to teamcity/.teamcity.vcs.ssh.json From 637c5ade2863627f66f26bd6ee354416c698e69d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 2581/2779] moved .concourse.yml to cicd/ --- .concourse.yml => cicd/.concourse.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .concourse.yml => cicd/.concourse.yml (100%) diff --git a/.concourse.yml b/cicd/.concourse.yml similarity index 100% rename from .concourse.yml rename to cicd/.concourse.yml From 424a095fedc3298d2ce34a63f6a4296b2acb8140 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 2582/2779] moved .gocd.yml to cicd/ --- .gocd.yml => cicd/.gocd.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .gocd.yml => cicd/.gocd.yml (100%) diff --git a/.gocd.yml b/cicd/.gocd.yml similarity index 100% rename from .gocd.yml rename to cicd/.gocd.yml From 2451e74fd432d4cdac907970369e2c9059608f3f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 2583/2779] moved buildspec.yml to cicd/ --- buildspec.yml => cicd/buildspec.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename buildspec.yml => cicd/buildspec.yml (100%) diff --git a/buildspec.yml b/cicd/buildspec.yml similarity index 100% rename from buildspec.yml rename to cicd/buildspec.yml From fca148ae783746b8452a26818344156c5a1bcdae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 2584/2779] moved cloudbuild.yaml to cicd/ --- cloudbuild.yaml => cicd/cloudbuild.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cloudbuild.yaml => cicd/cloudbuild.yaml (100%) diff --git a/cloudbuild.yaml b/cicd/cloudbuild.yaml similarity index 100% rename from cloudbuild.yaml rename to cicd/cloudbuild.yaml From caf90ee899ba2b558bdbab7d28c87ae28c5da7ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:41:46 +0100 Subject: [PATCH 2585/2779] removed shippable.yml --- shippable.yml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 shippable.yml diff --git a/shippable.yml b/shippable.yml deleted file mode 100644 index 1fa06bc06..000000000 --- a/shippable.yml +++ /dev/null @@ -1,50 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# http://docs.shippable.com/platform/workflow/config/ - -# http://docs.shippable.com/ci/advancedOptions/environmentVariables/ - -language: none - -branches: - only: - - master - -build: - ci: - # workaround to broken repos - # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB - # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 - # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed. - # DevOps-Python-tools/Makefile.in:272: recipe for target 'apt-packages' failed - - rm -fv -- /etc/apt/sources.list.d/cassandra.sources.list* - - rm -fv -- /etc/apt/sources.list.d/yarn.list* - # Basho repo is giving a '402 payment required' error - # https://github.com/Shippable/support/issues/5172 - - rm -fv -- /etc/apt/sources.list.d/basho_riak.list - #- shippable_retry make - - setup/ci_bootstrap.sh - - make init - - make ci - - make test - -integrations: - notifications: - - integrationName: email - type: email - on_success: never - on_failure: never - on_pull_request: never From 7d04cce80786cc6c4f278ce4a773f7d6a6f69ae3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:01 +0100 Subject: [PATCH 2586/2779] updated .appveyor.yml --- .appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index f4b36fba0..c459fd55a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A p p V e y o r C I +# ============================================================================ # + # https://www.appveyor.com/docs/appveyor-yml/ image: Ubuntu From 00b3cf9ae996e7b5a8f5bc7577adfd9b420b9381 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:02 +0100 Subject: [PATCH 2587/2779] updated pipeline.yml --- .buildkite/pipeline.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 93f047c02..eb0d641fd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u i l d K i t e C I +# ============================================================================ # + # BuildKite Pipeline # # add this command to the UI and it will read the rest of the steps from here: From 526798d5606da7d44e405fb9931b9d27c9c39b71 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:03 +0100 Subject: [PATCH 2588/2779] updated config.yml --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5dbde36b7..09026fc6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r c l e C I +# ============================================================================ # + # Master Template with more advanced config: # # https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml From 126ad67b87f9f2ada233929556a37d3ee479126e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:04 +0100 Subject: [PATCH 2589/2779] updated .cirrus.yml --- .cirrus.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index d544aeb85..6916e2577 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r r u s C I +# ============================================================================ # + # https://cirrus-ci.org/guide/writing-tasks/ container: From 243482c43161216eb1c278c64a0451f731347385 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:05 +0100 Subject: [PATCH 2590/2779] updated .drone.yml --- .drone.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.drone.yml b/.drone.yml index 4d36c9db2..d105ab57a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,6 +15,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# D r o n e C I +# ============================================================================ # + # https://docs.drone.io/quickstart/cli/ # # https://docs.drone.io/cli/install/ From 84d0b04dff32333bb7da8058de7c36b98bab6015 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:06 +0100 Subject: [PATCH 2591/2779] updated .gitlab-ci.yml --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 477b32c3e..625220b32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G i t L a b C I +# ============================================================================ # + # https://docs.gitlab.com/ee/ci/yaml/README.html +#include: '.gitlab/*.y*ml' + image: ubuntu:18.04 job: From c21088d44d170cf28533c2af5633c753b3a493ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:15:38 +0100 Subject: [PATCH 2592/2779] updated sonar-project.properties --- sonar-project.properties | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/sonar-project.properties b/sonar-project.properties index 7189b1a16..9b1539430 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon -# Date: 2016-07-19 18:18:27 +0100 (Tue, 19 Jul 2016) +# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016) # # https://github.com/HariSekhon/DevOps-Python-tools # @@ -12,21 +12,30 @@ # https://www.linkedin.com/in/HariSekhon # -sonar.projectName=DevOps Python Tools -sonar.projectKey=pytools +# ============================================================================ # +# S o n a r Q u b e +# ============================================================================ # + +sonar.host.url=https://sonarcloud.io + +# Required metadata +sonar.organization=harisekhon +sonar.projectName=DevOps-Python-tools +sonar.projectKey=HariSekhon_DevOps-Python-tools sonar.projectVersion=1.0 -sonar.projectDescription=Python / Jython Tools +sonar.projectDescription=DevOps-Python-tools sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions +# directories to scan (defaults to sonar-project.properties dir otherwise) sonar.sources=. #sonar.language=py sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/spark*/**/* +sonar.exclusions=**/tests/** From 2c93d263101b61483ad2189406aefe87efe53ecf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:39 +0100 Subject: [PATCH 2593/2779] updated sonar-project.properties --- sonar-project.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 9b1539430..319cc6897 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -38,4 +38,5 @@ sonar.sources=. sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/** +#sonar.exclusions=**/tests/** +sonar.exclusions=**/zookeeper-*/**/* From 3b6c0b0324ba01c6596adb589c86ad6790225fbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:50 +0100 Subject: [PATCH 2594/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e86c4f36e..761ef0697 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S e m a p h o r e C I +# ============================================================================ # + # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ version: v1.0 From 7e975e2cda81ac8b0177401bc8766caf7db9bf6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:51 +0100 Subject: [PATCH 2595/2779] updated README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 36f322455..35a30a739 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://harisekhon.github.io/CI-CD/) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) -[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) -[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.concourse.yml) +[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) @@ -55,8 +55,8 @@ [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) -[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) -[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) +[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) +[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) [![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) From 97a95b048ddff13a76f31a2687f62a8c3f4ee10f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:52 +0100 Subject: [PATCH 2596/2779] updated azure-pipelines.yml --- azure-pipelines.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 76ad8f1f0..e26e7f760 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A z u r e D e v O p s P i p e l i n e +# ============================================================================ # + # https://aka.ms/yaml trigger: From e7b73f1e14052b5e152d8934fddd83e11929db02 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:53 +0100 Subject: [PATCH 2597/2779] updated buddy.yml --- buddy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/buddy.yml b/buddy.yml index f901a6425..e0b57451f 100644 --- a/buddy.yml +++ b/buddy.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u d d y C I +# ============================================================================ # + # https://buddy.works/docs/yaml/yaml-schema --- From 3f82815fe59aae0a425f870296787d9206c161c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:59 +0100 Subject: [PATCH 2598/2779] updated .concourse.yml --- cicd/.concourse.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cicd/.concourse.yml b/cicd/.concourse.yml index 42b806f26..92847121e 100644 --- a/cicd/.concourse.yml +++ b/cicd/.concourse.yml @@ -13,6 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o n c o u r s e C I +# ============================================================================ # + +# https://concourse-ci.org/golang-library-example.html + +# https://resource-types.concourse-ci.org/ +# https://concourse-ci.org/resource-types.html resources: - name: github icon: github-circle @@ -25,8 +33,7 @@ resources: # source: # interval: 1d -# https://concourse-ci.org/golang-library-example.html - +# https://concourse-ci.org/jobs.html jobs: - name: build public: false From 3851ded922052a5aa98677a3045fcbca03a13aa3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:00 +0100 Subject: [PATCH 2599/2779] updated .gocd.yml --- cicd/.gocd.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cicd/.gocd.yml b/cicd/.gocd.yml index f3ed9fd19..65583a818 100644 --- a/cicd/.gocd.yml +++ b/cicd/.gocd.yml @@ -12,6 +12,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G o C D +# ============================================================================ # + # https://github.com/tomzo/gocd-yaml-config-plugin#setup # https://docs.gocd.org/current/configuration/configuration_reference.html From f7dda886aad2cdcd0374877ca6b5ab9dae4bd2d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:00 +0100 Subject: [PATCH 2600/2779] updated codefresh.yml --- codefresh.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codefresh.yml b/codefresh.yml index 5326f6e51..7cddcaf7c 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e f r e s h C I +# ============================================================================ # + # https://codefresh.io/docs/docs/codefresh-yaml/ version: "1.0" From 0865b14f7edd4a67a348af0139713a706fa67e3a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:01 +0100 Subject: [PATCH 2601/2779] updated kics.config --- kics.config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kics.config b/kics.config index 5fafd16d2..51ac5712e 100644 --- a/kics.config +++ b/kics.config @@ -30,7 +30,15 @@ log-file: true exclude-paths: # ignore submodules - handle them in the source repos only - bash-tools/ + - github-actions/ + - haproxy-configs/ + - jenkins/ + - kubernetes-templates/ + - lib/ - pylib/ + - spotify-tools/ - sql/ + - sql-keywords/ - templates/ + - terraform-templates/ #output-path: "results" From 0ba05249f4a515b998110738105078ef6cb4d003 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:02 +0100 Subject: [PATCH 2602/2779] updated gocd_config_repo.json --- setup/gocd_config_repo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json index 4c1639b4d..ce63bf63d 100644 --- a/setup/gocd_config_repo.json +++ b/setup/gocd_config_repo.json @@ -12,7 +12,7 @@ "configuration": [ { "key": "file_pattern", - "value": "*.gocd.y*ml" + "value": "cicd/*.gocd.y*ml" } ], "rules": [ From f605276d04c77181fa4e8cf71c85cf6002752a35 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:05 +0100 Subject: [PATCH 2603/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b0b1149f..36e0eabe2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b0b1149fabbbdb4a1c07cd25d2c1941034eb0a1 +Subproject commit 36e0eabe2b02207f51cd807fa6dafd18abdab234 From 37b79941c2329f2de041cab714d08851c1eaa5f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 2604/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e53f962a7..af52e8795 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e53f962a7d0eee7cf1f89ca98552d90c0d6226cb +Subproject commit af52e87958755d74d9b96cd373aab8cc810ab6a6 From 0d7d8060f515735afca206920e275643934e4867 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 2605/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index d4eedc0e2..b96c184fd 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit d4eedc0e2c68c9a43838c89a1d09abf10927a566 +Subproject commit b96c184fdda9b2ad607d222ec5663a087769b561 From d1ef74ffebc27e32ec9afeb5bca7debd4fedb64b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 2606/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3a74368ea..d58140c42 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3a74368ead1544983ac21af7500b380b9fc20b9f +Subproject commit d58140c424a83a440fa1480e878c45a00920c0f4 From a3634ecf1e18a624af24814f4540cba15623a642 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:32 +0100 Subject: [PATCH 2607/2779] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 6b9eaf3b9..4dc3a09c3 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -11,8 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e O w n e r s +# ============================================================================ # + --- -name: Codeowners +name: CodeOwners on: push: From 79fdd3ef11e57c87b703255fc61789aedd083080 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:33 +0100 Subject: [PATCH 2608/2779] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 4d4ebd577..c7ff16fd4 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k S y n c +# ============================================================================ # + +# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch + --- name: Fork Sync From 6900c0b4ca31f052e94e7caffb77c16ec8dd7938 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:34 +0100 Subject: [PATCH 2609/2779] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 00479ecb7..94398083f 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -11,6 +11,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k U p d a t e P R +# ============================================================================ # + +# For a fork of the original repo, activate to keep its branches up to date via Pull Requests +# +# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date + --- name: Fork Update PR From 02e4b41f7117802742d8af57107da9aa09677982 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:35 +0100 Subject: [PATCH 2610/2779] updated json.yaml --- .github/workflows/json.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index 6ed4b92d1..f68e309e5 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# J S O N +# ============================================================================ # + +# Validate any JSON files found in the repo + --- name: JSON From 77f1608b618c75c87ad51006dfddba709aabdf37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:35 +0100 Subject: [PATCH 2611/2779] updated kics.yaml --- .github/workflows/kics.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index 399cc40f0..033f4331e 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -11,6 +11,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# K i c s +# ============================================================================ # + --- name: Kics From 70a101b1ae1095ba49d05a182d3a77a7eb352f73 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:36 +0100 Subject: [PATCH 2612/2779] updated shellcheck.yaml --- .github/workflows/shellcheck.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 98e53bfe4..753a6a495 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S h e l l C h e c k +# ============================================================================ # + +# Validate any shell scripts found in the repo + --- name: ShellCheck From 55a9f677eb8afdb95879e413b15da0b3d5543b27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:36 +0100 Subject: [PATCH 2613/2779] updated trivy.yaml --- .github/workflows/trivy.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 8af2aeb39..26f01fa94 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -15,6 +15,8 @@ # T r i v y # ============================================================================ # +# Scan files in the local repo + --- name: Trivy From b5f76c8c6c099ca81c0e0d6885d41f1195d85379 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:37 +0100 Subject: [PATCH 2614/2779] updated validate.yaml --- .github/workflows/validate.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 5d924b77b..94a8c007a 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# V a l i d a t i o n +# ============================================================================ # + +# Run all custom validations against files in the repo + --- name: Validation From 88ac6b6661928c5f272515fe0700576d04b6f279 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:37 +0100 Subject: [PATCH 2615/2779] updated xml.yaml --- .github/workflows/xml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml index f4d3b7035..8b07ed256 100644 --- a/.github/workflows/xml.yaml +++ b/.github/workflows/xml.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# X M L +# ============================================================================ # + +# Validate any XML files found in the repo + --- name: XML From 2c992b85d84bb9dd9ebd7ea2e3ab57dfafe13ab2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:38 +0100 Subject: [PATCH 2616/2779] updated yaml.yaml --- .github/workflows/yaml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index bee8be84c..d252f67d2 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# Y A M L +# ============================================================================ # + +# Validate any YAML files found in the repo + --- name: YAML From a2a575ec7dec99394aab6b8572f42953129e3e43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:49 +0100 Subject: [PATCH 2617/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 36e0eabe2..815d75082 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 36e0eabe2b02207f51cd807fa6dafd18abdab234 +Subproject commit 815d75082d4faa8494e9500727ebd13e2c5abcae From 202d2ed1a86e2bd72f131fdd262eb6b88a846132 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 2618/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index af52e8795..ae8aa8753 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit af52e87958755d74d9b96cd373aab8cc810ab6a6 +Subproject commit ae8aa875359ddea2c9b5cf6ccc422e12a0740949 From 25abbfdedc8b9832c507f970f7c1a8ef29643a27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 2619/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index b96c184fd..e11486ebd 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit b96c184fdda9b2ad607d222ec5663a087769b561 +Subproject commit e11486ebd436802d22d1b88bda00c0a9eb3d41b6 From 24407a2473022336a734c95d262203780f75df87 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 2620/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d58140c42..3af953c9a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d58140c424a83a440fa1480e878c45a00920c0f4 +Subproject commit 3af953c9ae196d4ba5e59637e9668bd1bc6e9953 From 82da083e55bda834adf1802587856c6ff5b8b95c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:01 +0100 Subject: [PATCH 2621/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 815d75082..38901a555 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 815d75082d4faa8494e9500727ebd13e2c5abcae +Subproject commit 38901a555d2ce114f0e7df6d4a31a3fdc1a3dc41 From 4b432be8807c49caa2682b2f04ad492d403f2f0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:02 +0100 Subject: [PATCH 2622/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ae8aa8753..aa20abd72 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ae8aa875359ddea2c9b5cf6ccc422e12a0740949 +Subproject commit aa20abd72ceef27723ad8c461f4d9321ad12e779 From e14414287476521c71efa1ca3cb1095d2886ca62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:02 +0100 Subject: [PATCH 2623/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3af953c9a..15d1e3bc8 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3af953c9ae196d4ba5e59637e9668bd1bc6e9953 +Subproject commit 15d1e3bc8a594171b3c70ce0bf04979eceb8630a From baec6ac43c0a3bf03cd7d78bb35d9972d470c922 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 17:46:12 +0100 Subject: [PATCH 2624/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35a30a739..eb8e5fe12 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Perl Lib](https://github.com/HariSekhon/lib) - Perl version of above library -- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, ArgoCD, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... +- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python & D2 source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, Jenkins, ArgoCD, Traefik, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... From b5db3cb80db2ddc5dcc980a844939766452dc308 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:32:47 +0400 Subject: [PATCH 2658/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index da278a058..1438f07bd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit da278a05823c78709d831a4e6ba9cce4e5a481a7 +Subproject commit 1438f07bd9fe762752cfd986718e30797cff5650 From 8e50ccbc1c4ec668a1cea00152de7c700b584d24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:32:47 +0400 Subject: [PATCH 2659/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a43798fe9..6f9e99389 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a43798fe986928bffe4fe5952efdce408d012562 +Subproject commit 6f9e993896910c320c571918a635dc049e757e73 From a2d58fac6ac19b0d1b5f663aab5939eefe2b9f28 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:33:56 +0400 Subject: [PATCH 2660/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1438f07bd..b69edeada 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1438f07bd9fe762752cfd986718e30797cff5650 +Subproject commit b69edeadadf0774fc6444c552f3a89d407ca24d4 From 39a4591fa22b1a6c0ffe94e0f8451fa3a5a60f1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 May 2024 21:55:51 +0400 Subject: [PATCH 2661/2779] added .envrc --- .envrc | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..6fa553de2 --- /dev/null +++ b/.envrc @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# D i r e n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +# See more .envrc files in: +# +# https://github.com/HariSekhon/DevOps-Bash-tools +# +# .envrc-aws +# .envrc-gcp +# .envrc-kubernetes + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export COMPOSE_PROJECT_NAME="pytools" + +venv="$PWD/venv" + +if [ -f "$venv/bin/activate" ]; then + echo + echo "Local virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + echo +fi + +# read .env too +#dotenv From a0d2bc4abf677d283f992db3bab72461e888943f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:19:34 +0200 Subject: [PATCH 2662/2779] updated .envrc --- .envrc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.envrc b/.envrc index 6fa553de2..96d3b5c0f 100644 --- a/.envrc +++ b/.envrc @@ -35,18 +35,18 @@ set -euo pipefail export COMPOSE_PROJECT_NAME="pytools" -venv="$PWD/venv" - -if [ -f "$venv/bin/activate" ]; then - echo - echo "Local virtualenv directory found in: $venv" - echo - echo "Activating Virtualenv inside the directory: $venv" - - # shellcheck disable=SC1091 - source "$venv/bin/activate" - echo -fi +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + echo + fi +done # read .env too #dotenv From 0f9f99ff568ae21d903ceaaf0bf77b65c93632cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:20:41 +0200 Subject: [PATCH 2663/2779] updated .envrc --- .envrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.envrc b/.envrc index 96d3b5c0f..2aa9a4379 100644 --- a/.envrc +++ b/.envrc @@ -35,6 +35,7 @@ set -euo pipefail export COMPOSE_PROJECT_NAME="pytools" +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default for venv in "$PWD/venv" "$HOME/venv"; do if [ -f "$venv/bin/activate" ]; then echo From 92451e77812ddd6ace3c3ac401887fbeb2e1ad89 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:30 +0200 Subject: [PATCH 2664/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b69edeada..4cf683ab7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b69edeadadf0774fc6444c552f3a89d407ca24d4 +Subproject commit 4cf683ab744f849c34d2c15ce255b18b368d6a11 From eef5f927cf6bc15c6d438706a37567529347dd7d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:31 +0200 Subject: [PATCH 2665/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 99f5c866a..804fcd30f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 99f5c866aa5986c62d45de3b8da7043a28a63321 +Subproject commit 804fcd30fc9d05bf66fedae6001dbdef02dc8518 From 9616644dba9bf8d67673bcf02ae431af2395579d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:31 +0200 Subject: [PATCH 2666/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6f9e99389..a43798fe9 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6f9e993896910c320c571918a635dc049e757e73 +Subproject commit a43798fe986928bffe4fe5952efdce408d012562 From 7dd10775f06d708e16930d91519297addaa33519 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 2667/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4cf683ab7..8e8cf26f2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4cf683ab744f849c34d2c15ce255b18b368d6a11 +Subproject commit 8e8cf26f2c49b30371a35bbd844bae491fccfaf4 From aa3ef778e851dd980c962b4fafe2600dffd50123 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 2668/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 804fcd30f..c88505e98 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 804fcd30fc9d05bf66fedae6001dbdef02dc8518 +Subproject commit c88505e98b4a59ffa961ac92a581be96233d3b9d From 749724f6c95f2baba13490258a14b79b45c79f2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 2669/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a43798fe9..c2521be52 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a43798fe986928bffe4fe5952efdce408d012562 +Subproject commit c2521be525b4047222adb18263e5714e4171c483 From d8cbf436f9842d01a3bda55fdb095b2a4209b50d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:36:09 +0200 Subject: [PATCH 2670/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c88505e98..545ce8b83 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c88505e98b4a59ffa961ac92a581be96233d3b9d +Subproject commit 545ce8b838428cb8e3ae8e883130a5fe251bd83d From 21f534993310e30479c38661de27c0ffc350946e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:38:47 +0200 Subject: [PATCH 2671/2779] updated .envrc --- .envrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.envrc b/.envrc index 2aa9a4379..9073c82fd 100644 --- a/.envrc +++ b/.envrc @@ -14,7 +14,7 @@ # # ============================================================================ # -# D i r e n v +# P y t h o n D i r E n v # ============================================================================ # # .envrc to auto-load the virtualenv inside the 'venv' directory if present From 82154119348fccba6d13f674c336bdc127c4ac44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:39:28 +0200 Subject: [PATCH 2672/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8e8cf26f2..6d8f6ebc0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8e8cf26f2c49b30371a35bbd844bae491fccfaf4 +Subproject commit 6d8f6ebc0c93dad1306f35844b931c635cd1f9ce From 899a4cea79947f03e8e0862345914ddf47e68bee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:39:28 +0200 Subject: [PATCH 2673/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 545ce8b83..76e9e28d1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 545ce8b838428cb8e3ae8e883130a5fe251bd83d +Subproject commit 76e9e28d1b3a016dc64d1ccf92c7a967a74c48aa From a63287671e5471c97794e9c19f41b676d3d12e81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 2674/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6d8f6ebc0..9f1a8c533 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6d8f6ebc0c93dad1306f35844b931c635cd1f9ce +Subproject commit 9f1a8c53368a47e43e6f022f236e40635b78aa8f From 18cc287778458c4bfcaea0c12fca330302e6982c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 2675/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 76e9e28d1..e2982d2f4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 76e9e28d1b3a016dc64d1ccf92c7a967a74c48aa +Subproject commit e2982d2f4c07d0252d59897d9002235f6c0a7f6c From 1f6f01941062e293ee1c8898e884bafca470dccc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 2676/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 39d0eccb4..8597f692e 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 39d0eccb4c160fcee9f09df9bb53c911ca2a387f +Subproject commit 8597f692e4e4abd59b58fb24c7fa79449a75c8c0 From e805b2ab2933e6a05cfb0fbd0f6dfbabf5e50c3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 2677/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c2521be52..5174a5073 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c2521be525b4047222adb18263e5714e4171c483 +Subproject commit 5174a5073dbe25c8e5b1ac603b7b4f61b6ac7623 From 86d645a4f49915bd3032fcc9fee866f0a437eb94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:40:47 +0200 Subject: [PATCH 2678/2779] updated .envrc --- .envrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.envrc b/.envrc index 9073c82fd..51de39abb 100644 --- a/.envrc +++ b/.envrc @@ -46,6 +46,7 @@ for venv in "$PWD/venv" "$HOME/venv"; do # shellcheck disable=SC1091 source "$venv/bin/activate" echo + break fi done From c6fcf3037124d84f6f0b9ce569993fe41eb76181 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:41:19 +0200 Subject: [PATCH 2679/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9f1a8c533..f32bb01be 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9f1a8c53368a47e43e6f022f236e40635b78aa8f +Subproject commit f32bb01be9e59606bf4a292c0a3a7e2ef1ac4540 From 15816a21b449f0af739eb5985b61fe7ab9f9f475 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:41:19 +0200 Subject: [PATCH 2680/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e2982d2f4..29fbc0f2d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e2982d2f4c07d0252d59897d9002235f6c0a7f6c +Subproject commit 29fbc0f2da0c8d839898eb90b2d3a26c72ec85c8 From 959c4ce33775ecf588e4ad5794d1c340752510a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:33:25 +0200 Subject: [PATCH 2681/2779] updated .flake8 --- .flake8 | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index e31fb949f..de51ee708 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,32 @@ +# +# Author: Hari Sekhon +# Date: Mon Oct 21 15:57:10 2019 +0100 +# +# vim:ts=4:sts=4:sw=4:et +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# F l a k e 8 C o n f i g +# ============================================================================ # + +# https://flake8.pycqa.org/en/latest/user/configuration.html + [flake8] -ignore = E265,E402,F401 + max-line-length = 120 + +ignore = E265, + E402, + F401 + exclude = test*/* + max-complexity = 10 From 06d5e9469a49cf33c0c618ac5e05fbd4343dd4c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 2682/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f32bb01be..057b80266 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f32bb01be9e59606bf4a292c0a3a7e2ef1ac4540 +Subproject commit 057b80266f1bb0f3e36af2ae7f8fa228afc09074 From 49075c27e13a166fc5ac4dfd5582dfc9c936b928 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 2683/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 29fbc0f2d..fdb270866 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 29fbc0f2da0c8d839898eb90b2d3a26c72ec85c8 +Subproject commit fdb2708661b5a58fdba4b7ba514246aa929e6c02 From cd4e61aa6c5d4ff4181342e5d8f8581a1b135493 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 2684/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 5174a5073..d94994282 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 5174a5073dbe25c8e5b1ac603b7b4f61b6ac7623 +Subproject commit d94994282274abf4279444fd85c0674a382e6696 From ec92f71bad8c41778516345018fd6295861aa67f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:17:37 +0300 Subject: [PATCH 2685/2779] added .envrc-python --- .envrc-python | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .envrc-python diff --git a/.envrc-python b/.envrc-python new file mode 100644 index 000000000..6bc2d65b1 --- /dev/null +++ b/.envrc-python @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y t h o n D i r E n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + break + fi +done + +# read .env too +#dotenv From bae58f9cbadd09648d416c363f73e5ae1d5dd582 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:19:10 +0300 Subject: [PATCH 2686/2779] updated .envrc --- .envrc | 145 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 18 deletions(-) diff --git a/.envrc b/.envrc index 51de39abb..7e481411c 100644 --- a/.envrc +++ b/.envrc @@ -14,41 +14,150 @@ # # ============================================================================ # -# P y t h o n D i r E n v +# D i r E n v # ============================================================================ # -# .envrc to auto-load the virtualenv inside the 'venv' directory if present - # https://direnv.net/man/direnv-stdlib.1.html -# See more .envrc files in: -# -# https://github.com/HariSekhon/DevOps-Bash-tools +# See Also: # # .envrc-aws # .envrc-gcp # .envrc-kubernetes +# direnv stdlib - loads .envrc from parent dir up to / +# +# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. +# +# bypasses security authorization though - use with care +#source_up +# +# source_up must be loaded before set -u otherwise gets this error: +# +# direnv: loading .envrc +# /bin/bash: line 226: $1: unbound variable +# +# source_up causes this error is up .envrc is found in parent directories: +# +# direnv: No ancestor .envrc found + set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +src="$(readlink -f "${BASH_SOURCE[0]}")" +srcdir="$(cd "$(dirname "$src")" && pwd)" -export COMPOSE_PROJECT_NAME="pytools" +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # -# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default -for venv in "$PWD/venv" "$HOME/venv"; do - if [ -f "$venv/bin/activate" ]; then - echo - echo "Virtualenv directory found in: $venv" - echo - echo "Activating Virtualenv inside the directory: $venv" +# Automatically install Pre-Commit Git hooks if not already present - # shellcheck disable=SC1091 - source "$venv/bin/activate" +if [ -f .pre-commit-config.yaml ]; then + if [ -d .git ]; then + if ! [ -f .git/hooks/pre-commit ]; then + pre-commit install + fi + fi +fi + +# ============================================================================ # +# D o c k e r C o m p o s e +# ============================================================================ # + +export COMPOSE_PROJECT_NAME="DevOps-Python-tools" + +# ============================================================================ # +# G i t H u b +# ============================================================================ # + +#export GITHUB_ORGANIZATION=HariSekhon + +# ============================================================================ # +# A n s i b l e +# ============================================================================ # + +# use the local repo's ansible.cfg rather than: +# +# $PWD/ansible.cfg +# ~/.ansible.cfg +# /etc/ansible/ansible.cfg +# +# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used +#export ANSIBLE_CONFIG="/path/to/ansible.cfg" + +# ============================================================================ # +# C l o u d f l a r e +# ============================================================================ # + +#export CLOUDFLARE_EMAIL=hari@... +#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens +#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module + +# export the variables for terraform +#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" +#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above + +# ============================================================================ # +# P y t h o n , A W S , G C P , T e r r a f o r m +# ============================================================================ # + +# XXX: safer to bring all these external .envrc inline if you're worried about changes +# to it bypassing 'direnv allow' authorization +load_if_exists(){ + # first arg is a path to a .envrc + # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes + # to pass the context name 'docker-desktop' to switch to + local envrc="$1" + shift + if ! [[ "$envrc" =~ ^/ ]]; then + envrc="$srcdir/$envrc" + fi + if [ -f "$envrc" ]; then + # prevent looping on symlinks to this .envrc if given + if [ "$(readlink "$envrc")" = "$src" ]; then + return + fi echo - break + echo "Loading $envrc" + # shellcheck disable=SC1090,SC1091 + . "$envrc" "$@" fi +} + +#load_if_exists ~/.envrc + + #.envrc-aws \ + #.envrc-gcp \ + #.envrc-terraform \ +# shellcheck disable=SC2043 +for envrc in \ + .envrc-python \ + ; do + load_if_exists "$envrc" done +if [[ "$PWD" =~ /aws/ ]]; then + load_if_exists .envrc-aws +fi + +if [[ "$PWD" =~ /gcp/ ]]; then + load_if_exists .envrc-gcp +fi + +if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then + load_if_exists .envrc-terraform +fi + +# ============================================================================ # +# K u b e r n e t e s +# ============================================================================ # + +if [ -f "$srcdir/.envrc-kubernetes" ]; then + load_if_exists .envrc-kubernetes docker-desktop +fi + +# ============================================================================ # + +echo # read .env too #dotenv From 13b3cde4939345bbb49086f16b627056958b2329 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:51:24 +0300 Subject: [PATCH 2687/2779] added .pre-commit-config.yaml --- .pre-commit-config.yaml | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..f3c5b1608 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,72 @@ +# +# Author: Hari Sekhon +# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024) +# +# vim:ts=2:sts=2:sw=2:et +# +# https///github.com/HariSekhon/Templates +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +--- +fail_fast: false +#exclude: *.tmp$ + +repos: + + # will accept anything that 'git clone' understands + # this means you can set this to a local git repo to develop your own hook repos interactively + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + # Common errors + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: forbid-new-submodules + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + + - repo: https://github.com/psf/black + rev: 24.8.0 + hooks: + - id: black + + # Git secrets Leaks + - repo: https://github.com/awslabs/git-secrets.git + # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error: + # + # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file + # + rev: 5357e18 + hooks: + - id: git-secrets + + - repo: https://github.com/markdownlint/markdownlint + rev: v0.12.0 + hooks: + - id: markdownlint + name: Markdownlint + description: Run markdownlint on your Markdown files + entry: mdl + language: ruby + files: \.(md|mdown|markdown)$ From 5c145c7d2e6459c61a9a55c0f6f7047e009ccbda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:57:50 +0300 Subject: [PATCH 2688/2779] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3c5b1608..2b154867f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https///github.com/HariSekhon/Templates +# https///github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # From c8c6623a14f7482ab9b222f63074f0bcbb55e657 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 20:33:29 +0300 Subject: [PATCH 2689/2779] updated .envrc --- .envrc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.envrc b/.envrc index 7e481411c..da56f607a 100644 --- a/.envrc +++ b/.envrc @@ -52,11 +52,11 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present -if [ -f .pre-commit-config.yaml ]; then - if [ -d .git ]; then - if ! [ -f .git/hooks/pre-commit ]; then - pre-commit install - fi +if [ -f .pre-commit-config.yaml ] && + [ -d .git ] && + type -P pre-commit &>/dev/null; then + if ! [ -f .git/hooks/pre-commit ]; then + pre-commit install fi fi From 9a5c385d480ecf1e05aacf4e0e753af684f9e262 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 00:12:38 +0300 Subject: [PATCH 2690/2779] updated .editorconfig --- .editorconfig | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 73c19b202..462f53206 100644 --- a/.editorconfig +++ b/.editorconfig @@ -40,7 +40,14 @@ end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true -[*.md] +[{*.md,*.hcl,*.tf,*.tfvars}] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.yml,*.yaml] indent_size = 2 indent_style = space end_of_line = lf From f410868fe774c96117e790f2bb7f6f2bd3fd9af6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 01:49:04 +0300 Subject: [PATCH 2691/2779] updated .envrc --- .envrc | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index da56f607a..920b4e16d 100644 --- a/.envrc +++ b/.envrc @@ -98,7 +98,7 @@ export COMPOSE_PROJECT_NAME="DevOps-Python-tools" #export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above # ============================================================================ # -# P y t h o n , A W S , G C P , T e r r a f o r m +# Load External Envrc Files If Present # ============================================================================ # # XXX: safer to bring all these external .envrc inline if you're worried about changes @@ -124,8 +124,15 @@ load_if_exists(){ fi } +# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc +# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses +# to install to system Python) #load_if_exists ~/.envrc +# ============================================================================ # +# P y t h o n +# ============================================================================ # + #.envrc-aws \ #.envrc-gcp \ #.envrc-terraform \ @@ -136,14 +143,26 @@ for envrc in \ load_if_exists "$envrc" done +# ============================================================================ # +# A W S +# ============================================================================ # + if [[ "$PWD" =~ /aws/ ]]; then load_if_exists .envrc-aws fi +# ============================================================================ # +# G C P +# ============================================================================ # + if [[ "$PWD" =~ /gcp/ ]]; then load_if_exists .envrc-gcp fi +# ============================================================================ # +# T e r r a f o r m +# ============================================================================ # + if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then load_if_exists .envrc-terraform fi From 283a9a3ca15ea2293c9ae2ed08a9ac77aa7d0bfa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 12:06:01 +0300 Subject: [PATCH 2692/2779] updated .envrc --- .envrc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.envrc b/.envrc index 920b4e16d..15bc469ee 100644 --- a/.envrc +++ b/.envrc @@ -52,10 +52,23 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present +if ! type -P pre-commit &>/dev/null && + uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now..." + echo + brew install pre-commit + echo +fi + if [ -f .pre-commit-config.yaml ] && - [ -d .git ] && - type -P pre-commit &>/dev/null; then - if ! [ -f .git/hooks/pre-commit ]; then + type -P pre-commit &>/dev/null && + git rev-parse --is-inside-work-tree &>/dev/null; then + if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + echo + echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." + echo pre-commit install fi fi From f09fdd3279da8103197fc4be455be86eea23e2df Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 14 Aug 2024 23:17:32 +0200 Subject: [PATCH 2693/2779] updated .envrc --- .envrc | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.envrc b/.envrc index 15bc469ee..7555ecba9 100644 --- a/.envrc +++ b/.envrc @@ -52,14 +52,20 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present -if ! type -P pre-commit &>/dev/null && - uname -s | grep -q Darwin && - type -P brew &>/dev/null; then - echo - echo "Pre-commit is not installed - installing now..." - echo - brew install pre-commit - echo +if ! type -P pre-commit &>/dev/null; then + if uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Homebrew..." + echo + brew install pre-commit + echo + elif type -P pip &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Pip..." + echo + pip install pre-commit + fi fi if [ -f .pre-commit-config.yaml ] && @@ -188,6 +194,8 @@ if [ -f "$srcdir/.envrc-kubernetes" ]; then load_if_exists .envrc-kubernetes docker-desktop fi +# ============================================================================ # +# . E n v # ============================================================================ # echo From d31b9e28d672eedc9624cfc2890daf982fd6acd9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 15 Aug 2024 16:35:10 +0200 Subject: [PATCH 2694/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 503347f5d..cc918aa8d 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,11 @@ [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) [![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) -[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) + [![ShellCheck](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml) [![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) From f3c5690d32c077c52b108ca1f83f88fbae06cfbe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 22 Aug 2024 02:22:09 +0200 Subject: [PATCH 2695/2779] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b154867f..df14fb7d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,5 +68,6 @@ repos: name: Markdownlint description: Run markdownlint on your Markdown files entry: mdl + args: [-s, .markdownlint.rb] language: ruby files: \.(md|mdown|markdown)$ From 4546c7688be6a4e0a12b333b8fa123c0aae7a91c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 22 Aug 2024 02:22:10 +0200 Subject: [PATCH 2696/2779] added .markdownlint.rb --- .markdownlint.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .markdownlint.rb diff --git a/.markdownlint.rb b/.markdownlint.rb new file mode 100644 index 000000000..e69de29bb From db58f5309bcf6fe5d145bb65afdc501b0c1494d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:18 +0200 Subject: [PATCH 2697/2779] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df14fb7d9..9e9c1c291 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,6 +68,6 @@ repos: name: Markdownlint description: Run markdownlint on your Markdown files entry: mdl - args: [-s, .markdownlint.rb] + args: [-s, .mdl.rb] language: ruby files: \.(md|mdown|markdown)$ From 0839b2b170aa75b94c39263577660b3cac5e25ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:19 +0200 Subject: [PATCH 2698/2779] added .mdlrc --- .mdlrc | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .mdlrc diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 000000000..27e5b6895 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,5 @@ +mdlrc_dir = File.expand_path('..', __FILE__) + +style_file = File.join(mdlrc_dir, '.mdl.rb') + +style style_file From c8ec774ef167b959685256791528eba64dbd8b32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:19 +0200 Subject: [PATCH 2699/2779] added .mdl.rb --- .mdl.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .mdl.rb diff --git a/.mdl.rb b/.mdl.rb new file mode 100644 index 000000000..db6a3373f --- /dev/null +++ b/.mdl.rb @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# vim:ts=4:sts=4:sw=4:et:filetype=ruby +# +# Author: Hari Sekhon +# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +all +#exclude_rule 'MD001' +#exclude_rule 'MD003' +#exclude_rule 'MD005' +exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af +#exclude_rule 'MD012' +exclude_rule 'MD013' # long lines cannot be split if they are URLs +#exclude_rule 'MD022' +#exclude_rule 'MD025' +#exclude_rule 'MD031' +#exclude_rule 'MD032' +exclude_rule 'MD033' # inline HTML is important for formatting +#exclude_rule 'MD039' +#exclude_rule 'MD056' From cfa61d4d041dba2306dfdfbf568226209d656f56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:36:37 +0200 Subject: [PATCH 2700/2779] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index ab20cfa8d..0b52378ff 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -52,7 +52,7 @@ retry(){ if [ "$(uname -s)" = Darwin ]; then echo "Bootstrapping Mac" # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make) - #retry "$srcdir/install_homebrew.sh" + #retry "$srcdir/../install/install_homebrew.sh" if command -v brew 2>&1; then # fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do From 63bde7fef5cf084fa7fe4a0a212d60d275f46ef2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 2701/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 057b80266..e7c73c3c4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 057b80266f1bb0f3e36af2ae7f8fa228afc09074 +Subproject commit e7c73c3c436c41901d1b51c872e5c3e2b22e2b07 From 16a294f584f532cde95b2df7401cc50234bdd3d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 2702/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fdb270866..092819070 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fdb2708661b5a58fdba4b7ba514246aa929e6c02 +Subproject commit 092819070eda268f0a4c6f8b373290c09a9de421 From c91a96c1ebd6a698d568d6a36d758872da9804b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 2703/2779] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 8597f692e..ace6d6400 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 8597f692e4e4abd59b58fb24c7fa79449a75c8c0 +Subproject commit ace6d6400c1f647f2e96bd6dc9b730cf1232847a From b53766eb78df127afdcc81e3dde0a50a8f9dabe8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:20 +0200 Subject: [PATCH 2704/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d94994282..b8d565217 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d94994282274abf4279444fd85c0674a382e6696 +Subproject commit b8d565217a111514a2e061cec896f258712985fb From 82e9390f13636322f39891aa5cc372a4496428c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 17:15:58 +0200 Subject: [PATCH 2705/2779] updated README --- README.md | 196 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 116 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index cc918aa8d..4fd1177dc 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) -[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/HariSekhon/) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) -- [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... +### Knowledge -- [The Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) -- [Nagios Plugin Kafka](https://github.com/HariSekhon/Nagios-Plugin-Kafka) - Kafka API pub/sub Nagios Plugin written in Scala with Kerberos support + -- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python & D2 source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, Jenkins, ArgoCD, Traefik, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... +### DevOps Code -- [Knowledge-Base](https://github.com/HariSekhon/Knowledge-Base) - IT Knowledge Base from 20 years in DevOps, Linux, Cloud, Big Data, AWS, GCP etc. +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Python-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Perl-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Perl-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Golang-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Golang-tools) -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +### Containerization -- https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) -### Stargazers over time +### CI/CD -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) -[git.io/python-tools](https://git.io/python-tools) +### DBA - SQL -[git.io/pytools](https://git.io/pytools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) + +### DevOps Reloaded + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) + +### Misc + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) + +The rest of my original source repos are +[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). + +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). + + +![](https://hit.yhype.me/github/profile?user_id=2211051) + + From e27764e4a9ed13170d3585259ea6267be3b7f62d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 17:17:33 +0200 Subject: [PATCH 2706/2779] updated README --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4fd1177dc..5fa8938af 100644 --- a/README.md +++ b/README.md @@ -534,14 +534,17 @@ Does nothing: [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) -### Misc +### Templates +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) + +### Misc + [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) From 6259d5a9e744abde66b0ff91c15b194e8104b075 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:19:58 +0200 Subject: [PATCH 2707/2779] updated .mdl.rb --- .mdl.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/.mdl.rb b/.mdl.rb index db6a3373f..f8f9b004c 100644 --- a/.mdl.rb +++ b/.mdl.rb @@ -25,5 +25,6 @@ #exclude_rule 'MD031' #exclude_rule 'MD032' exclude_rule 'MD033' # inline HTML is important for formatting +exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines #exclude_rule 'MD039' #exclude_rule 'MD056' From e4eac61b95c830eb6ee80027214d72e9be9dcd6e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:30:55 +0200 Subject: [PATCH 2708/2779] updated README --- README.md | 176 +++++++++++++++++++++++++++--------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 5fa8938af..955eb8cb6 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Cloud & Big Data Contractor, United Kingdom [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/)
*(you're welcome to connect with me on LinkedIn)* -**Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries** +**Make sure you run `make update` if updating and not just `git pull` as you will often need the latest library submodule and possibly new upstream libraries** ## Quick Start @@ -163,14 +163,14 @@ Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://git ### Usage -All programs come with a ```--help``` switch which includes a program description and the list of command line options. +All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. ### DevOps Python Tools - Inventory - Linux: - - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -179,115 +179,115 @@ Environment variables are supported for convenience and also to hide credentials - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` - - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - ```find_active_server.py``` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` + - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [Amazon Web Services](https://aws.amazon.com/): - - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all docker images eg. `docker_pull_all_images_all_tags.sh` + - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` - [Spark](https://spark.apache.org/) & Data Format Converters: - - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter - - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas - - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```xml_to_json.py``` - XML to JSON converter - - ```json_to_xml.py``` - JSON to XML converter - - ```json_to_yaml.py``` - JSON to YAML converter - - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - - ```yaml_to_json.py``` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) - - see also ```validate_*.py``` further down for all these formats and more + - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter + - `spark_parquet_to_avro.py` - PySpark Parquet => Avro converter + - `spark_csv_to_avro.py` - PySpark CSV => Avro converter, supports both inferred and explicit schemas + - `spark_csv_to_parquet.py` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas + - `spark_json_to_avro.py` - PySpark JSON => Avro converter + - `spark_json_to_parquet.py` - PySpark JSON => Parquet converter + - `xml_to_json.py` - XML to JSON converter + - `json_to_xml.py` - JSON to XML converter + - `json_to_yaml.py` - JSON to YAML converter + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) + - see also `validate_*.py` further down for all these formats and more - [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: - [Ambari](https://hortonworks.com/apache/ambari/): - - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API + - `ambari_blueprints.py` - Blueprint cluster templating and deployment tool using Ambari API - list blueprints - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool - - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API - - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable using this tool + - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts + - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API + - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - ```hdfs_find_replication_factor_1.py``` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - ```hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - ```hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output The following programs can all optionally filter by database / table name regex: - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables - - ```hive_tables_row_column_counts.py``` / ```impala_tables_row_column_counts.py``` - outputs tables row and column counts. Useful for finding unusually big tables - - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_metadata.py``` / ```impala_tables_metadata.py``` - outputs for each table the matching regex metadata DDL property from describe table - - ```hive_tables_locations.py``` / ```impala_tables_locations.py``` - outputs for each table its data location + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL property from describe table + - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables - - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop -- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig_udfs.jy` - Pig Jython UDFs for Hadoop +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required - - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node + - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA + - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA + - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node + - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator + - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server + - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server + - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node + - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- ```selenium_hub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - - ```validate_*.py``` - validate files, directory trees and/or standard input streams + - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: - Avro - CSV @@ -326,7 +326,7 @@ Download the DevOps Python Tools and Pylib git repos as zip files: -Unzip both and move Pylib to the ```pylib``` folder under DevOps Python Tools. +Unzip both and move Pylib to the `pylib` folder under DevOps Python Tools. ```shell unzip devops-python-tools-master.zip @@ -450,7 +450,7 @@ make update This will git pull and then git submodule update which is necessary to pick up corresponding library updates. -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run ```make update-no-recompile``` (will miss new library dependencies - do full ```make update``` if you encounter issues). +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter issues). ### Testing @@ -476,9 +476,9 @@ You might also be interested in the following really nice Jupyter notebook for H -### Stargazers over time +## Star History -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) +[![Star History Chart](https://api.star-history.com/svg?repos=HariSekhon/DevOps-Python-tools&type=Date)](https://star-history.com/#HariSekhon/DevOps-Python-tools&Date) [git.io/python-tools](https://git.io/python-tools) From 73ff2ef8fe71271639bea252a671c6a70fbf5830 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:32:22 +0200 Subject: [PATCH 2709/2779] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 955eb8cb6..8c3ad36f1 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,7 @@ Environment variables are supported for convenience and also to hide credentials - [Hadoop](http://hadoop.apache.org/) HDFS: - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing `hdfs dfs -cat | md5sum`) - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output From 94d58f49cb50bbe8ef645d3a785914cd6231b6ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 20:48:32 +0200 Subject: [PATCH 2710/2779] updated README.md --- README.md | 253 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 181 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 8c3ad36f1..f9fdde42d 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,8 @@ cd pytools make ``` -To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension instead of `.py`: +To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension +instead of `.py`: ```shell make anonymize.pyc @@ -165,12 +166,15 @@ Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://git All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list +eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and +often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. ### DevOps Python Tools - Inventory - Linux: - - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing + - lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -181,27 +185,57 @@ Environment variables are supported for convenience and also to hide credentials - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) - - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes + communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support + teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel + before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation + important for anonymization rules, as well as maintaining file content order. On servers this parallelization can + result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file + basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, + useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). + Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See + further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in + your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [Amazon Web Services](https://aws.amazon.com/): - - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally + filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have + never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. + Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for + users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to + [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is + triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and + [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP + address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its + accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via + VPC connector routing + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with + their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` - - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all docker images eg. `docker_pull_all_images_all_tags.sh` + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a + docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very + useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only + the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests + across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official + `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned + to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy + shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. + `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all + docker images eg. `docker_pull_all_images_all_tags.sh` - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` - [Spark](https://spark.apache.org/) & Data Format Converters: - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter @@ -213,7 +247,13 @@ Environment variables are supported for convenience and also to hide credentials - `xml_to_json.py` - XML to JSON converter - `json_to_xml.py` - JSON to XML converter - `json_to_yaml.py` - JSON to YAML converter - - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for + pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and + [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / + directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard + output for convenient command line chaining and redirection, optionally continues on error, collects broken records + to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not + technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) - see also `validate_*.py` further down for all these formats and more - [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: @@ -223,69 +263,119 @@ Environment variables are supported for convenience and also to hide credentials - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control + - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for + proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable using this tool + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable + using this tool - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing `hdfs dfs -cat | md5sum`) - - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to + replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file + or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data + layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster + than doing `hdfs dfs -cat | md5sum`) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree + showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format + to standard output The following programs can all optionally filter by database / table name regex: - - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala table - - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation between cluster migrations - - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for finding unusually wide tables - - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. Useful for finding unusually big tables - - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL property from describe table + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala + table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation + between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for + finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. + Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where + any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or + subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. + Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain + NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL + property from describe table - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting tables - - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, + with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI + tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting + tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, + giving per region row counts and % of total rows for the table as well as median and quartile row counts per + regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in + an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row + counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating + on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an + optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or + memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average + since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / + total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or + average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, + write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular + RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received + the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew + and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase + tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - `pig_udfs.jy` - Pig Jython UDFs for Hadoop -- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, + useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex + content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for + which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you + want to query cluster wide information available from any online peer (eg. Elasticsearch) + - The following are simplified specialisations of the above program, just pass host arguments, all the details have + been baked in, no switches required - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run + multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server + (run multiple of these for load balancing) - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, + Catalog or Statestore - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) + - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the + [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - + useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks + session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot + debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as + Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: @@ -297,14 +387,20 @@ Environment variables are supported for convenience and also to hide credentials - Parquet - XML - YAML - - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) - - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies + - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, + `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) + - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for + things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images + [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & + Linux technologies ### Detailed Build Instructions #### Python VirtualEnv localized installs -The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages instead of installing from PyPI then follow the Manual Build section below. +The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it +detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages +instead of installing from PyPI then follow the Manual Build section below. ### Manual Setup @@ -337,7 +433,8 @@ mv -v pylib-master pylib mv -vf pylib pytools/ ``` -Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). +Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal +mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files. @@ -355,9 +452,11 @@ sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt #### Mac OS X -The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing `git` is enough to trigger Xcode install). +The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing +`git` is enough to trigger Xcode install). -I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): +I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for +development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): ```shell bash-tools/install/install_homebrew.sh @@ -373,7 +472,8 @@ If failing to build an OpenSSL lib dependency, just prefix the build command lik sudo OPENSSL_INCLUDE=/usr/local/opt/openssl/include OPENSSL_LIB=/usr/local/opt/openssl/lib ... ``` -You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is caused by pip 10 vs pip 9 and downgrading will work around it: +You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is +caused by pip 10 vs pip 9 and downgrading will work around it: ```shell sudo pip install --upgrade pip==9.0.1 @@ -398,7 +498,8 @@ Run like so: jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` -The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. +The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop +APIs. See below for procedure to install Jython if you don't already have it. @@ -422,7 +523,10 @@ Then add the Jython install bin directory to the $PATH or specify the full path ### Configuration for Strict Domain / FQDN validation -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my +[PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, +`.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal +environments). ### Python SSL certificate verification problems @@ -433,7 +537,8 @@ If you end up with an error like: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:765) ``` -It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One quick fix is to do the following: +It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One +quick fix is to do the following: ```shell sudo pip uninstall -y certifi && @@ -450,7 +555,9 @@ make update This will git pull and then git submodule update which is necessary to pick up corresponding library updates. -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter issues). +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each +time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter +issues). ### Testing @@ -466,13 +573,15 @@ To trigger all tests run: make test ``` -which will start with the underlying libraries, then move on to top level integration tests and functional tests using docker containers if docker is available. +which will start with the underlying libraries, then move on to top level integration tests and functional tests using +docker containers if docker is available. ### Contributions Patches, improvements and even general feedback are welcome in the form of GitHub pull requests and issue tickets. -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another +Hortonworks guy Jonas Straub: From 68cdf7de64f0e0e6193c99ad330dcd4e92e6b4f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Sep 2024 01:17:17 +0200 Subject: [PATCH 2711/2779] removed .markdownlint.rb --- .markdownlint.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .markdownlint.rb diff --git a/.markdownlint.rb b/.markdownlint.rb deleted file mode 100644 index e69de29bb..000000000 From 2ff0c6b46c0cb830261565739e20e365dad717e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 18:40:41 +0100 Subject: [PATCH 2712/2779] updated azure-pipelines.yml --- azure-pipelines.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e26e7f760..b430c58dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,19 +22,26 @@ trigger: - master +variables: + # ubuntu version + os_version: '22.04' + pool: # there is no /dev/stderr on this azure build! #vmImage: 'ubuntu-latest' - # Ubuntu 16.04 required for docker container support, looks like 18.04 works too - vmImage: 'ubuntu-22.04' + #vmImage: 'ubuntu-22.04' + vmImage: 'ubuntu-$(os_version)' # unprivileged container without sudo, cannot install dependencies -#container: ubuntu:18.04 +#container: ubuntu:22.04 steps: + - script: cat /etc/*-release + displayName: OS Release + # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' - script: env | sort - displayName: env + displayName: Environment # doesn't work in container due to unprivileged execution and lack of sudo #- script: sudo apt-get update && sudo apt-get install -y git make @@ -50,7 +57,7 @@ steps: # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( - script: | - sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c ' + sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c ' set -ex cd /code setup/ci_bootstrap.sh From def50a98abb78cca6c938bb9fcca1ac660fde7a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 2713/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e7c73c3c4..399bc52b2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e7c73c3c436c41901d1b51c872e5c3e2b22e2b07 +Subproject commit 399bc52b2b9fa0d3aa4b60d2eb18cb1e3b3ae0b6 From 38c02005e8cd21b410901588698693ea6f26eebf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 2714/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 092819070..81714c62c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 092819070eda268f0a4c6f8b373290c09a9de421 +Subproject commit 81714c62ccfee06feef0ab8cef5c1485bff7d9c0 From 566d308bd84fa3a7ff327d7f05f67a83bb05c0ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 2715/2779] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ace6d6400..575080b54 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ace6d6400c1f647f2e96bd6dc9b730cf1232847a +Subproject commit 575080b548fec8b2420520e161973a861cbbce30 From fbca030c08ef4b3af3a131252dfbef8459bdd031 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 2716/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index b8d565217..da9d4c1ab 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit b8d565217a111514a2e061cec896f258712985fb +Subproject commit da9d4c1ab5787bbd3802db78a2330ad713ee3cbe From b77d999a26ca7f335eccc397881d3981536d8dc0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:26 +0100 Subject: [PATCH 2717/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 399bc52b2..4c661ca88 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 399bc52b2b9fa0d3aa4b60d2eb18cb1e3b3ae0b6 +Subproject commit 4c661ca889cbef24e989f27c184d12d02cb99c98 From c6ab1cc4353d5a55bff3b41a775635537a9c5f7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:26 +0100 Subject: [PATCH 2718/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 81714c62c..666501ada 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 81714c62ccfee06feef0ab8cef5c1485bff7d9c0 +Subproject commit 666501adaf43473a0c94b2ace714598a80cdb549 From 1981197b571c7f661bc912648878f6ef46cf37ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:27 +0100 Subject: [PATCH 2719/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index da9d4c1ab..6cee5eb76 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit da9d4c1ab5787bbd3802db78a2330ad713ee3cbe +Subproject commit 6cee5eb764acd9f9198bd83f9af3b17bf49e1283 From 3e40bd427b4780b74718a7dbdf395cf33484a814 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:12 +0100 Subject: [PATCH 2720/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4c661ca88..3dd58df8c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4c661ca889cbef24e989f27c184d12d02cb99c98 +Subproject commit 3dd58df8cbaeaf22661c2cf68f289535508c4685 From 5b7e885adc73fada6c37497d18fabc7c1290c47f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:13 +0100 Subject: [PATCH 2721/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 666501ada..c90dd3e3a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 666501adaf43473a0c94b2ace714598a80cdb549 +Subproject commit c90dd3e3ae1ccb64ed0e668750fa594b3c709563 From bff7195cde897972a547360784d5a0f7129fd559 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:13 +0100 Subject: [PATCH 2722/2779] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6cee5eb76..2d12574ce 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6cee5eb764acd9f9198bd83f9af3b17bf49e1283 +Subproject commit 2d12574ceafbcb13f559dfa46693024616b8e619 From c1a5672830bd8eb9628f505a305186cbbd65100c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:34:26 +0100 Subject: [PATCH 2723/2779] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f377f523f..80ff670d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ GitPython==2.1.15 #happybase==1.0.0 humanize==0.5.1 -impyla==0.16.0 +impyla==0.19.0 jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 From 5c900849a8d38f989a1343c3c36d54d7026adb03 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:38:31 +0100 Subject: [PATCH 2724/2779] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3dd58df8c..adc18db20 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3dd58df8cbaeaf22661c2cf68f289535508c4685 +Subproject commit adc18db20909106dbf4d6ededa1c6c66d5c28e8c From f373795d24110882a5e8a036484dfa0d5a423ded Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:38:31 +0100 Subject: [PATCH 2725/2779] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c90dd3e3a..eb10979e3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c90dd3e3ae1ccb64ed0e668750fa594b3c709563 +Subproject commit eb10979e3572ef37b35bda3653b25d7a45e25169 From 2e739a43d79ff855a58c1d76725065c7d1257d34 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 22:12:45 +0100 Subject: [PATCH 2726/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index adc18db20..6fc486cfb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit adc18db20909106dbf4d6ededa1c6c66d5c28e8c +Subproject commit 6fc486cfb85886c8b125c3649b80de1762eaba21 From 84e699a8bdae550158a4940234c98996666efbbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Sep 2024 11:52:18 +0100 Subject: [PATCH 2727/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6fc486cfb..176f383c9 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6fc486cfb85886c8b125c3649b80de1762eaba21 +Subproject commit 176f383c9148361e04dfde142054ce2efbdbb5e6 From b580060646cdba0f99af7e8bb557a53491d223a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Sep 2024 12:22:58 +0100 Subject: [PATCH 2728/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 176f383c9..34f300de0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 176f383c9148361e04dfde142054ce2efbdbb5e6 +Subproject commit 34f300de0e37f5cba5176796560c99e3780a7cc1 From c9f8ba3236600b8666053f3c9fc7bdbcfeff7dd2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Sep 2024 00:07:20 +0100 Subject: [PATCH 2729/2779] updated .envrc --- .envrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.envrc b/.envrc index 7555ecba9..bafe8df0b 100644 --- a/.envrc +++ b/.envrc @@ -201,3 +201,5 @@ fi echo # read .env too #dotenv + +load_if_exists .envrc.local From 1b5dcec3b4c4ca4373615075b6db22c589a4d870 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2024 09:27:46 +0300 Subject: [PATCH 2730/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 34f300de0..4f4be21ed 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 34f300de0e37f5cba5176796560c99e3780a7cc1 +Subproject commit 4f4be21ed3006a756971134afa71c283388b111d From ee32cb23b8c4617015737f9c4938c852c0c03f19 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Oct 2024 18:35:23 +0300 Subject: [PATCH 2731/2779] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e9c1c291..f615a7725 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,14 +30,14 @@ repos: hooks: - id: check-yaml # Common errors - - id: end-of-file-fixer + #- id: end-of-file-fixer # ruins .gitignore Icon\r - id: trailing-whitespace args: [--markdown-linebreak-ext=md] # Git style - id: check-added-large-files - id: check-merge-conflict - id: check-vcs-permalinks - - id: forbid-new-submodules + #- id: forbid-new-submodules # Cross platform - id: check-case-conflict - id: mixed-line-ending From f04002186cf34e64c7a7503d91144fa7521df597 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 11 Oct 2024 17:51:20 +0300 Subject: [PATCH 2732/2779] added markdown.yaml --- .github/workflows/markdown.yaml | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/markdown.yaml diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml new file mode 100644 index 000000000..cb5395963 --- /dev/null +++ b/.github/workflows/markdown.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# M a r k D o w n +# ============================================================================ # + +--- +name: Markdown + +on: + push: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + Markdown: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Markdown + uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master From 0e7f2891dca92120dc47009e1e496a871b8ac3cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 11 Oct 2024 18:00:17 +0300 Subject: [PATCH 2733/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f9fdde42d..6d11affff 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ [![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) +[![Markdown](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) From ac683274c27147af43de231a903b2e6945bd63b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:29:36 +0400 Subject: [PATCH 2734/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4f4be21ed..956f47624 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4f4be21ed3006a756971134afa71c283388b111d +Subproject commit 956f47624a5fcefdf46c5d675298e0526121713c From b9a12fcc4a414016b44da2af0cd060f5b97dbe57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:32:27 +0400 Subject: [PATCH 2735/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 956f47624..cf080c2c5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 956f47624a5fcefdf46c5d675298e0526121713c +Subproject commit cf080c2c584de729b3bbbcefb3cfa353cf64216a From 5206c0ff64ecd8f3b08d050859be1b6080011631 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:47:40 +0400 Subject: [PATCH 2736/2779] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f615a7725..6ae7e968c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,10 +46,10 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] - - repo: https://github.com/psf/black - rev: 24.8.0 - hooks: - - id: black + #- repo: https://github.com/psf/black + # rev: 24.8.0 + # hooks: + # - id: black # Git secrets Leaks - repo: https://github.com/awslabs/git-secrets.git From ea2ed7d241ea06422fea231a2fcc513b17597ab6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:47:41 +0400 Subject: [PATCH 2737/2779] permitted space in section headers to allow for AWS [profile blah] sections --- validate_ini.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_ini.py b/validate_ini.py index 54a01cc63..b38b3ab6a 100755 --- a/validate_ini.py +++ b/validate_ini.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.12.2' +__version__ = '0.12.3' class IniValidatorTool(CLI): @@ -72,7 +72,7 @@ def __init__(self): self.re_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I) # In Windows ini key cannot contain equals sign = or semicolon ; # key=val or [section] - self.re_ini_section = re.compile(r'^\s*\[([\w=\:\.-]+)\]\s*$') + self.re_ini_section = re.compile(r'^\s*\[([\w\s=\:\.-]+)\]\s*$') self.re_ini_key = re.compile(r'^\s*(?:[^\[;=]+)s*$') # INI value can be anything .* so not regex'ing it self.valid_ini_msg = ' => INI OK' From 67c680b35148ee03a249575ed54a0c047ead4144 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:52:07 +0400 Subject: [PATCH 2738/2779] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ae7e968c..787fb79ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,6 +46,7 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] + # rewrites python files with useless changes like changing single quotes to double quotes #- repo: https://github.com/psf/black # rev: 24.8.0 # hooks: From 510fd69c3f7c7ee0a98ad81ae26123c8f7fb1be9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 24 Nov 2024 04:06:10 +0400 Subject: [PATCH 2739/2779] updated test_dockerhub_search.sh --- tests/test_dockerhub_search.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index 92e3d2e9d..2a4f9c48e 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -36,7 +36,7 @@ check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED # shellcheck disable=SC2016 -check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" +check '[ "$(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep -c "^harisekhon/[A-Za-z0-9_-]*$")" = 40 ]' "DockerHub Search quiet mode for shell scripting" echo echo From 22fa4cff9849d3230cbbb48a1733c6403286510b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Dec 2024 00:03:50 +0700 Subject: [PATCH 2740/2779] updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09026fc6a..ab6c6499d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,7 +37,7 @@ workflows: jobs: build: docker: - - image: cimg/base:2021.04 + - image: cimg/base:2024.12 resource_class: small steps: - checkout From 6b56bd27261b489177dece866bc49ca7fdd2364a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 2 Feb 2025 00:21:59 +0700 Subject: [PATCH 2741/2779] added lint exception for on: truthy --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/codeowners.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_11.yaml | 2 +- .github/workflows/debian_12.yaml | 2 +- .github/workflows/docker_pytools_alpine.yaml | 2 +- .github/workflows/docker_pytools_debian.yaml | 2 +- .github/workflows/docker_pytools_fedora.yaml | 2 +- .github/workflows/docker_pytools_ubuntu.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/grype.yaml | 2 +- .github/workflows/json.yaml | 2 +- .github/workflows/kics.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_11.yaml | 2 +- .github/workflows/mac_12.yaml | 2 +- .github/workflows/markdown.yaml | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.11.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/shellcheck.yaml | 2 +- .github/workflows/trivy.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_22.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/validate.yaml | 2 +- .github/workflows/xml.yaml | 2 +- .github/workflows/yaml.yaml | 2 +- 38 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a678d6592..dc5dfa86c 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -14,7 +14,7 @@ --- name: Alpine -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index fd88e54cb..a522558aa 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -14,7 +14,7 @@ --- name: Alpine 3 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 8f31b2672..209b3cef1 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -22,7 +22,7 @@ --- name: Checkov -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 4dc3a09c3..6a0edd459 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -18,7 +18,7 @@ --- name: CodeOwners -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d56453263..7794a2914 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -14,7 +14,7 @@ --- name: Debian -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 735603e28..e8fec2ae1 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -14,7 +14,7 @@ --- name: Debian 10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml index de3b4b96f..3bebff0c9 100644 --- a/.github/workflows/debian_11.yaml +++ b/.github/workflows/debian_11.yaml @@ -14,7 +14,7 @@ --- name: Debian 11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml index d543710b1..3dbcdd303 100644 --- a/.github/workflows/debian_12.yaml +++ b/.github/workflows/debian_12.yaml @@ -14,7 +14,7 @@ --- name: Debian 12 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index 769f90e49..7119d6852 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Alpine) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index b2b718ac0..24f8f21dd 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Debian) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 70c545db6..ad1a5ed8d 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Fedora) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index e69018c31..884917919 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Ubuntu) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 0975be19d..3dfae11a4 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -14,7 +14,7 @@ --- name: Fedora -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index c7ff16fd4..3c1aecadf 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -20,7 +20,7 @@ --- name: Fork Sync -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 94398083f..bedec7659 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,7 +22,7 @@ --- name: Fork Update PR -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml index 6a321f86e..351180e94 100644 --- a/.github/workflows/grype.yaml +++ b/.github/workflows/grype.yaml @@ -18,7 +18,7 @@ --- name: Grype -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index f68e309e5..83fa5ef83 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -20,7 +20,7 @@ --- name: JSON -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index 033f4331e..f7f7383fe 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -18,7 +18,7 @@ --- name: Kics -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index ac6957e87..4959ef720 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -14,7 +14,7 @@ --- name: Mac -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml index 2d948e136..64ec035b6 100644 --- a/.github/workflows/mac_11.yaml +++ b/.github/workflows/mac_11.yaml @@ -14,7 +14,7 @@ --- name: Mac 11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml index 7d2bf2dc6..5fa32d10c 100644 --- a/.github/workflows/mac_12.yaml +++ b/.github/workflows/mac_12.yaml @@ -14,7 +14,7 @@ --- name: Mac 12 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml index cb5395963..72e301ba0 100644 --- a/.github/workflows/markdown.yaml +++ b/.github/workflows/markdown.yaml @@ -18,7 +18,7 @@ --- name: Markdown -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index de6f79440..7d953dfd7 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -14,7 +14,7 @@ --- name: Python 3.10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml index 8e2d041ac..91658c8e9 100644 --- a/.github/workflows/python3.11.yaml +++ b/.github/workflows/python3.11.yaml @@ -14,7 +14,7 @@ --- name: Python 3.11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 58ad6c210..04c7146c7 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -14,7 +14,7 @@ --- name: Python 3.7 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index f1c806dc2..37ad93654 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -14,7 +14,7 @@ --- name: Python 3.8 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 31c7d0ab8..9cf18e290 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -14,7 +14,7 @@ --- name: Python 3.9 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 43b3f5bd2..3ac4c40b9 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -20,7 +20,7 @@ --- name: Semgrep Cloud -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index f1e452f88..94fd43a22 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -22,7 +22,7 @@ --- name: Semgrep -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 753a6a495..ea4a967eb 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -20,7 +20,7 @@ --- name: ShellCheck -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 26f01fa94..c31ec991e 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -20,7 +20,7 @@ --- name: Trivy -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 0d7fd952c..8edea5c2d 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 5145177bb..ae5ec97ef 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu 20.04 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml index b9bf17a95..d5dfb94a8 100644 --- a/.github/workflows/ubuntu_22.04.yaml +++ b/.github/workflows/ubuntu_22.04.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu 22.04 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 53307dd73..2efa3ac6d 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -14,7 +14,7 @@ --- name: GitHub Actions Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 94a8c007a..468d875e3 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -20,7 +20,7 @@ --- name: Validation -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml index 8b07ed256..3f6265a6a 100644 --- a/.github/workflows/xml.yaml +++ b/.github/workflows/xml.yaml @@ -20,7 +20,7 @@ --- name: XML -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index d252f67d2..5c3118c33 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -20,7 +20,7 @@ --- name: YAML -on: +on: # yamllint disable-line rule:truthy push: branches: - master From 7738761d337f64222906c6a623ed410dfb18fa44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 2 Feb 2025 00:23:39 +0700 Subject: [PATCH 2742/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cf080c2c5..a97248860 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cf080c2c584de729b3bbbcefb3cfa353cf64216a +Subproject commit a97248860172c6bb3534d08ad38c4cfdfd09c306 From 406fce6a86160038a5ca122f54d0a401fab1e93b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2025 20:35:59 +0700 Subject: [PATCH 2743/2779] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d11affff..22741ad41 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) +[![LineCount](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2)](https://github.com/boyter/scc/) +[![Cocomo](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2&category=cocomo)](https://github.com/boyter/scc/) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) From c9f6bce6e1972482bd39a67a793c6c241398933a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2025 14:00:14 +0700 Subject: [PATCH 2744/2779] changed concurrency lock to be workflow-ref format --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml.disabled | 2 +- .github/workflows/centos7.yaml.disabled | 2 +- .github/workflows/centos8.yaml.disabled | 2 +- .github/workflows/codeowners.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_11.yaml | 2 +- .github/workflows/debian_12.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml.disabled | 2 +- .github/workflows/debian_9.yaml.disabled | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/grype.yaml | 2 +- .github/workflows/kics.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_11.yaml | 2 +- .github/workflows/mac_12.yaml | 2 +- .github/workflows/pypy2.yaml.disabled | 2 +- .github/workflows/pypy3.yaml.disabled | 2 +- .github/workflows/python2.7.yaml.disabled | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.11.yaml | 2 +- .github/workflows/python3.6.yaml.disabled | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/shellcheck.yaml | 2 +- .github/workflows/trivy.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml.disabled | 2 +- .github/workflows/ubuntu_16.04.yaml.disabled | 2 +- .github/workflows/ubuntu_18.04.yaml.disabled | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_22.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/validate.yaml | 2 +- .github/workflows/yaml.yaml | 2 +- 44 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index dc5dfa86c..9bed93efb 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index a522558aa..f6c4f5ea9 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos.yaml.disabled b/.github/workflows/centos.yaml.disabled index ef6aace7b..420b88392 100644 --- a/.github/workflows/centos.yaml.disabled +++ b/.github/workflows/centos.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos7.yaml.disabled b/.github/workflows/centos7.yaml.disabled index a891a3dbb..ec764d288 100644 --- a/.github/workflows/centos7.yaml.disabled +++ b/.github/workflows/centos7.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos8.yaml.disabled b/.github/workflows/centos8.yaml.disabled index d39fc2b0a..76c476664 100644 --- a/.github/workflows/centos8.yaml.disabled +++ b/.github/workflows/centos8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 6a0edd459..168d4914c 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 7794a2914..d84cb16b1 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index e8fec2ae1..e857f280f 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml index 3bebff0c9..ad765f1ad 100644 --- a/.github/workflows/debian_11.yaml +++ b/.github/workflows/debian_11.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml index 3dbcdd303..dfbc4ec97 100644 --- a/.github/workflows/debian_12.yaml +++ b/.github/workflows/debian_12.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 1cb4e5431..f3062b6f8 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index e987d7e46..1b0e00295 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_8.yaml.disabled b/.github/workflows/debian_8.yaml.disabled index 37640f59e..b41ead7d3 100644 --- a/.github/workflows/debian_8.yaml.disabled +++ b/.github/workflows/debian_8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_9.yaml.disabled b/.github/workflows/debian_9.yaml.disabled index 42e28702b..451348091 100644 --- a/.github/workflows/debian_9.yaml.disabled +++ b/.github/workflows/debian_9.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 3dfae11a4..5a4f9a9b4 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 3c1aecadf..6b228ca33 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -34,7 +34,7 @@ permissions: contents: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index bedec7659..c74fa2a0b 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -37,7 +37,7 @@ permissions: pull-requests: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml index 351180e94..1b0fa15b4 100644 --- a/.github/workflows/grype.yaml +++ b/.github/workflows/grype.yaml @@ -46,7 +46,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index f7f7383fe..c389f5df0 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -46,7 +46,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 4959ef720..438dd184e 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml index 64ec035b6..cc392b105 100644 --- a/.github/workflows/mac_11.yaml +++ b/.github/workflows/mac_11.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml index 5fa32d10c..0bcd6e6fd 100644 --- a/.github/workflows/mac_12.yaml +++ b/.github/workflows/mac_12.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pypy2.yaml.disabled b/.github/workflows/pypy2.yaml.disabled index 7dbc25c7f..01080bcc8 100644 --- a/.github/workflows/pypy2.yaml.disabled +++ b/.github/workflows/pypy2.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pypy3.yaml.disabled b/.github/workflows/pypy3.yaml.disabled index ac669813c..101b36751 100644 --- a/.github/workflows/pypy3.yaml.disabled +++ b/.github/workflows/pypy3.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python2.7.yaml.disabled b/.github/workflows/python2.7.yaml.disabled index ff3e51dd1..1813931e7 100644 --- a/.github/workflows/python2.7.yaml.disabled +++ b/.github/workflows/python2.7.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 7d953dfd7..8ec3008b9 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml index 91658c8e9..0f27b0076 100644 --- a/.github/workflows/python3.11.yaml +++ b/.github/workflows/python3.11.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.6.yaml.disabled b/.github/workflows/python3.6.yaml.disabled index 84dce85e0..91bd3e673 100644 --- a/.github/workflows/python3.6.yaml.disabled +++ b/.github/workflows/python3.6.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 04c7146c7..73a0045c7 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 37ad93654..abcbc4c68 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 9cf18e290..1ce7eb238 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 3ac4c40b9..c2f614222 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 94fd43a22..04a3cf197 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -50,7 +50,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index ea4a967eb..246087bd4 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index c31ec991e..95b2571ca 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -48,7 +48,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 8edea5c2d..fe863f875 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_14.04.yaml.disabled b/.github/workflows/ubuntu_14.04.yaml.disabled index 395184d7b..1442d675d 100644 --- a/.github/workflows/ubuntu_14.04.yaml.disabled +++ b/.github/workflows/ubuntu_14.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_16.04.yaml.disabled b/.github/workflows/ubuntu_16.04.yaml.disabled index 0a55dd214..a9655835d 100644 --- a/.github/workflows/ubuntu_16.04.yaml.disabled +++ b/.github/workflows/ubuntu_16.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml.disabled index d909f9529..4b510ba1e 100644 --- a/.github/workflows/ubuntu_18.04.yaml.disabled +++ b/.github/workflows/ubuntu_18.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index ae5ec97ef..47d0512ac 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml index d5dfb94a8..ba8a549b1 100644 --- a/.github/workflows/ubuntu_22.04.yaml +++ b/.github/workflows/ubuntu_22.04.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 2efa3ac6d..0037e4ba3 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 468d875e3..ab6ab9638 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index 5c3118c33..f2a4ca129 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -50,7 +50,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: From 5dbc2ec8cd49f8990767878568ae893ba7b4dd5b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2025 16:31:03 +0700 Subject: [PATCH 2745/2779] updated .cirrus.yml --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 6916e2577..471dc55ba 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -23,8 +23,8 @@ container: image: ubuntu:18.04 task: - # doesn't work properly - #skip: "!changesInclude('**/*.md')" + env: + TMPDIR: /var/tmp script: - setup/ci_bootstrap.sh - make init From 10d0ad24c89cededf549e38c49f209234d90825b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:05:48 +0700 Subject: [PATCH 2746/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 761ef0697..e2b78c511 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -23,8 +23,8 @@ version: v1.0 name: DevOps-Python-tools agent: machine: - type: e1-standard-2 - os_image: ubuntu1804 + type: e2-standard-2 + os_image: ubuntu2204 execution_time_limit: hours: 3 blocks: From 0adb1596606a13a7e3ee873bacd76953d4c6d39f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:28:56 +0700 Subject: [PATCH 2747/2779] updated config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab6c6499d..b22b98dbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,8 +41,8 @@ jobs: resource_class: small steps: - checkout - - setup_remote_docker: - version: 20.10.11 + #- setup_remote_docker: + # version: 20.10.11 - run: setup/ci_bootstrap.sh - run: make init - run: make From c10d63286ad086d0fb4e3b403f823a38aa7699a0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:56:31 +0700 Subject: [PATCH 2748/2779] updated config.yml --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b22b98dbb..312bf1020 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,6 +40,11 @@ jobs: - image: cimg/base:2024.12 resource_class: small steps: + # CLI is too old - config validate breaks in test - install new version to fix + # doesn't work - existing version is too old to update + #- run: circleci update + - run: | + curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash - checkout #- setup_remote_docker: # version: 20.10.11 From d8d48a9fc94d856d27c805b56c2287e727debedf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:04:54 +0700 Subject: [PATCH 2749/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e2b78c511..308278eee 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,9 +22,10 @@ version: v1.0 name: DevOps-Python-tools agent: + # https://docs.semaphoreci.com/reference/machine-types machine: - type: e2-standard-2 - os_image: ubuntu2204 + type: e1-standard-2 + os_image: ubuntu2004 execution_time_limit: hours: 3 blocks: From 3204b856ac322c4c2e3c5b30bbf562484dae5fa7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:14:01 +0700 Subject: [PATCH 2750/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 308278eee..e8de9f5e5 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,6 +44,9 @@ blocks: prologue: commands: - cache restore + - export DEBIAN_FRONTEND=noninteractive + - sudo apt-get update + - sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From 2895d5f6ecc4012b3eded89bdb4990f5e41c3e92 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:21:10 +0700 Subject: [PATCH 2751/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e8de9f5e5..fdb60732e 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,9 +44,14 @@ blocks: prologue: commands: - cache restore - - export DEBIAN_FRONTEND=noninteractive - - sudo apt-get update - - sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server + - sudo rm -f /etc/ssh/sshd_config + #- export DEBIAN_FRONTEND=noninteractive + #- sudo apt-get update + #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections + #- sudo dpkg --configure -a --force-confdef --force-confold + #- sudo apt-get install -y openssh-server -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From 99cf65b7fef1586302c8f0979c4cda90aac1ce43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:32:25 +0700 Subject: [PATCH 2752/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fdb60732e..5d5a7ab20 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -45,13 +45,20 @@ blocks: commands: - cache restore # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server - - sudo rm -f /etc/ssh/sshd_config - #- export DEBIAN_FRONTEND=noninteractive - #- sudo apt-get update + # + # causes error: + # + # Not replacing deleted config file /etc/ssh/sshd_config + # + #- sudo rm -f /etc/ssh/sshd_config + - export DEBIAN_FRONTEND=noninteractive + - sudo -E apt-get update + - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew" + - sudo dpkg --configure -a --force-confmiss --force-confnew #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections #- sudo dpkg --configure -a --force-confdef --force-confold - #- sudo apt-get install -y openssh-server -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + - sudo apt-get install -y openssh-server # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From f21e0aba680245a800caf478d4b248c8e3870e62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2025 02:07:53 +0700 Subject: [PATCH 2753/2779] updated semaphore.yml --- .semaphore/semaphore.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5d5a7ab20..59aa80836 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,7 +22,7 @@ version: v1.0 name: DevOps-Python-tools agent: - # https://docs.semaphoreci.com/reference/machine-types + # https://docs.semaphoreci.com/reference/machine-types#linux machine: type: e1-standard-2 os_image: ubuntu2004 @@ -85,9 +85,10 @@ blocks: #- name: DEBUG # value: "1" agent: + # https://docs.semaphoreci.com/reference/machine-types#macos machine: type: a1-standard-4 - os_image: macos-xcode12 + os_image: macos-xcode15 prologue: commands: - cache restore From 1ad7d99584197c2cc80e91d03e6dc335fa2606f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2025 21:38:06 +0700 Subject: [PATCH 2754/2779] updated .envrc --- .envrc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index bafe8df0b..e4488d51f 100644 --- a/.envrc +++ b/.envrc @@ -71,7 +71,16 @@ fi if [ -f .pre-commit-config.yaml ] && type -P pre-commit &>/dev/null && git rev-parse --is-inside-work-tree &>/dev/null; then - if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" + if [ -L "$hook" ]; then + echo "Detected symlink hook: " + echo + ls -l "$hook" + echo + echo "Removing" + rm -f "$hook" + fi + if ! [ -f "$hook" ]; then echo echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." echo From 6b1938824a0bf6ef7a278c2f08061801dbae7ad9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:41:33 +0800 Subject: [PATCH 2755/2779] updated bootstrap.sh --- setup/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 94b10159c..d7bedf92a 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -45,11 +45,11 @@ elif [ "$(uname -s)" = Linux ]; then if [ -n "${CI:-}" ]; then export DEBIAN_FRONTEND=noninteractive fi - opts="" + opts="-o DPkg::Lock::Timeout=1200" if [ -z "${PS1:-}" ]; then - opts="-qq" + opts="$opts -qq" fi - $sudo apt-get update $opts + $sudo apt-get update $opts $sudo apt-get install $opts -y git make curl wget --no-install-recommends elif type yum >/dev/null 2>&1; then if grep -qi 'NAME=.*CentOS' /etc/*release; then From 2efb7e7dad0b6979452cc4d229e50e72e32bdf9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:41:34 +0800 Subject: [PATCH 2756/2779] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index 0b52378ff..b01a77d04 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -72,8 +72,9 @@ elif [ "$(uname -s)" = Linux ]; then retry $sudo apk update retry $sudo apk add --no-progress bash git make elif type apt-get >/dev/null 2>&1; then - retry $sudo apt-get update -q - retry $sudo apt-get install -qy git make + opts="-q -o DPkg::Lock::Timeout=1200" + retry $sudo apt-get update $opts + retry $sudo apt-get install $opts -y git make elif type yum >/dev/null 2>&1; then #retry $sudo yum makecache retry $sudo yum install -qy git make From b13623f40359ddd19ff96f6c2ae78a56375b3222 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:52:41 +0800 Subject: [PATCH 2757/2779] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a435bc5a5..2a53008d4 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,7 @@ apk-packages-extra: .PHONY: apt-packages-extra apt-packages-extra: - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y default-jdk; fi + if [ -z "$(NOJAVA)" ]; then which java || bash-tools/packages/apt_install_packages.sh default-jdk; fi # for validate_multimedia.py # available in Alpine 2.6, 2.7 and 3.x From 0bba44755a7b874dfc582aa7e15d59c7a94d0986 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:54:28 +0800 Subject: [PATCH 2758/2779] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2a53008d4..d4434ca84 100644 --- a/Makefile +++ b/Makefile @@ -189,8 +189,8 @@ apk-packages-multimedia: # Debian 9 Stretch onwards, not available in Debian 8 Jessie .PHONY: apt-packages-multimedia apt-packages-multimedia: - $(SUDO) apt-get update - $(SUDO) apt-get install -y --no-install-recommends ffmpeg + $(SUDO) apt-get update -o DPkg::Lock::Timeout=1200 + $(SUDO) apt-get install -o DPkg::Lock::Timeout=1200 -y --no-install-recommends ffmpeg # for validate_multimedia.py .PHONY: yum-packages-multimedia From 72d01f4ef7dc4646eb3a35d665dce0936a445b65 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:55:27 +0800 Subject: [PATCH 2759/2779] updated Makefile --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d4434ca84..1d98ac524 100644 --- a/Makefile +++ b/Makefile @@ -200,9 +200,10 @@ yum-packages-multimedia: .PHONY: jython jython: - if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi - if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi - if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + @#if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi + @#if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi + @#if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + bash-tools/packages/install_packages.sh wget expect sh jython_install.sh .PHONY: test-lib From feb48b7a21ea137bc18ca92344cdb8a5e75d98bd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:56:01 +0800 Subject: [PATCH 2760/2779] updated test_validate_multimedia.sh --- tests/test_validate_multimedia.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index c86449aac..3493bae6d 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -32,7 +32,7 @@ if ! type -P ffmpeg &>/dev/null; then echo "WARNING: ffmpeg not installed, skipping validate_multimedia.py tests" exit 0 if type -P apt-get &>/dev/null; then - sudo apt-get install -y ffmpeg + sudo apt-get install -o DPkg::Lock::Timeout=1200 -y ffmpeg elif type -P yum &>/dev/null; then echo "WARNING: cannot auto-install ffmpeg on RHEL/CentOS, the 3rd party repos and deps are seriously broken" fi From 55835218d3367e90d76d6a0614727ede7351e1f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:38:30 +0800 Subject: [PATCH 2761/2779] added plot_uk_marriage_rates.py --- plot_uk_marriage_rates.py | 158 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 plot_uk_marriage_rates.py diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py new file mode 100755 index 000000000..6b1297417 --- /dev/null +++ b/plot_uk_marriage_rates.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2025-04-26 00:37:28 +0800 (Sat, 26 Apr 2025) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Plots the Marriage rates in England & Wales using the latest download from the UK Government website: + + https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/marriagecohabitationandcivilpartnerships/bulletins/marriagesinenglandandwalesprovisional/2021and2022 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import platform +import subprocess +import sys +#import time +import traceback +import pandas as pd +import matplotlib.pyplot as plt +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +# pylint: disable=too-few-public-methods +class PlotUKMarriageRates(CLI): + + def __init__(self): + # Python 2.x + super(PlotUKMarriageRates, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = 0 + + # def add_options(self): + # super(PlotUKMarriageRates, self).add_options() + # + # def process_options(self): + # super(PlotUKMarriageRates, self).process_options() + + def run(self): + if not self.args: + self.usage("Provide path to datadownload.xlsx") + + file_path = self.args[0] + + if not os.path.isfile(file_path): + self.usage(f"Invalid argument provided, not a file: {file_path}") + + log.info(f"Loading file: {file_path}") + xls = pd.ExcelFile(file_path) + + log.info("Parsing xls") + # Read the relevant sheet and skip the metadata + df = xls.parse('Figure 2', skiprows=6) + + # Set proper column headers + df.columns = df.iloc[1] + df = df[2:] # Drop the header rows + + # Rename columns for clarity + df.columns = ['Year', 'Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women'] + + # Convert data types + df['Year'] = df['Year'].astype(int) + for col in ['Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women']: + df[col] = pd.to_numeric(df[col], errors='coerce') + + log.info("Plotting") + plt.figure(figsize=(10, 6)) + plt.plot(df['Year'], df['Opposite-sex Men'], label='Opposite-sex Men') + plt.plot(df['Year'], df['Opposite-sex Women'], label='Opposite-sex Women') + plt.plot(df['Year'], df['Same-sex Men'], label='Same-sex Men') + plt.plot(df['Year'], df['Same-sex Women'], label='Same-sex Women') + + plt.title('Marriage Rates Over Time (England & Wales)') + plt.xlabel('Year') + plt.ylabel('Marriage Rate per 1,000 People') + plt.legend() + plt.grid(True) + plt.tight_layout() + #plt.show() + # + # doesn't show anything because the script finishes before the GUI event loop + # has time to process and display the window + #plt.show(block=False) + # + # pylint: disable=line-too-long + # + # doesn't work even with this hack: + # + # WARNING: NSWindow geometry should only be modified on the main thread! This will raise an exception in the future + # + #import threading + #threading.Thread(target=plt.show).start() + #sleep_secs = 20 + #log.info(f"Sleeping for {sleep_secs} secs to allow you to see the graph pop-up") + #time.sleep(sleep_secs) + + image_path = os.path.splitext(file_path)[0] + '.png' + log.info("Generating output image: {image_path}") + + #if os.path.exists(image_path): + # log.warning(f"Image page already exists, skipping recreating for safety: {image_path}") + # #plt.close() + # return + + # doesn't solve blank png + #plt.gcf().canvas.draw() + + # results in blank png + #plt.savefig(image_path) + + fig = plt.gcf() # draw before saving + fig.canvas.draw() # force rendering + plt.savefig(image_path) + + plt.show() + + if platform.system() == "Darwin": + # fire and forget + # pylint: disable=subprocess-run-check + subprocess.run(['open', image_path]) + + #plt.close() + + +if __name__ == '__main__': + PlotUKMarriageRates().main() From 1a0723e81e31574595e06f5cb7f85038aaf033a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:39:20 +0800 Subject: [PATCH 2762/2779] updated requirements.txt --- requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.txt b/requirements.txt index 80ff670d0..05d204c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,11 +36,17 @@ ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 +matplotlib==3.7.5 + # Python 3.5+ #numpy==1.17.2 # XXX: install broken on new M1 Mac Python 3.9 #numpy==1.16.5 +# for plot_uk_marriage_rates.py +openpyxl==3.1.5 +pandas==2.0.3 + # requires pg_config to build from source #psycopg2==2.8.4 # XXX: install broken on new M1 Mac Python 3.9 From 53def06b018590b505dc43379c0b27e99d5e1fd2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:40:48 +0800 Subject: [PATCH 2763/2779] added uk_marriage_rates_2022.xslx --- tests/data/uk_marriage_rates_2022.xslx | Bin 0 -> 15337 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/data/uk_marriage_rates_2022.xslx diff --git a/tests/data/uk_marriage_rates_2022.xslx b/tests/data/uk_marriage_rates_2022.xslx new file mode 100644 index 0000000000000000000000000000000000000000..54efc33218a9d57c8f26797118393a120865a840 GIT binary patch literal 15337 zcmeHugGM3^)Wj2nYxaun~hS!31!ngMxq{K!bpw0so^TVrT1YX6tOA z>S1r@q(|>=V_mBzBhw>@+z$5?TBTu8PaP(yg2W$9UA*oxDxY3>Z~V68xWnzsXOgMV zy36?30q>Yw^Y|{n*M&_4)U0*nV znNfCN*;>ks*Vqzsm~ER^Ruk@fCs!U8Jj_w)7yG08ovGzSZWeb4J<}US99$5``p6+h z6Hf>d_7GuufUKbd&s1<}JWUOwwQMhGeS1Kz&zHs7ypyagNpX?09Ml!6fmOiQ;p^Bp zmBg~cmqbhTZyJi7hi?>CMsAuak*Gws_fV}X*J~7PTo{;<$(X;-ey)qmxF1W-#ivE^ z)H(6|fWAET1OF?-5T??uuHw%0Sr&_(1QmE?M1~7;C5!XG3b|<&LXe~rT;u7h@tpJ& z+$2E(R6ojoFU;@wUYXIh^7J+wG#vV&)2eeIT>&!Y*@ydO4vPhtX2K%xDZy?1IgsjMtc9-IcAxpRBO30d1(Am)VTx11{O zz026jjPyy`K|upZG1FLC;no(Jo4gG>`>LXLQw?v|8lis4+7PU)!6@CX_Tk`wVOs z9y7~nofbe;`M5u1IAsm!`Yy27oqaVO5aVG#3%0>2qZ1ukn*H|Rl9k`d$}P4OKbL1_ zauMvZ^RYDBuX!2f_6KEe0w{`NzqBT%q z68cV;%yQ9fBsVomoKq{E;n% zUSP$%fki%#TDrI#Pg0U@wWIYNBRcMtk?_@F!7f`Fx36Jnk&8>>MLDZmbRT)s$l50U%zGZeT0vMybI9Xkd6rBcWqEr z6K6QAzLls=QKsV3s~!VEV_d8bHsSdO(t`7Soi@I`B6ahar0@ey;kVj45IFi(i;r12 z0DnzPPH68NR5jr;&30ii}w`@+B52?HQDIrqEe)HIe38&FDa@R$8e601?t-;JO zazQqnTn7Az96g(S3~G>5xLov{)*RPxTcxmACIgps#|8PRQKR!;$EV05z_)jLMS91; z(BsJj^Jytw5$6(eZ*&fNN?353_%GUXEOudX8C=F=+gl^lh@{GihbOjQ-asVT7DPtlxjK#|;=vVK378N9#pZe^tW4qL?~~lvEOJMK z@<{j=upYek{dSU;hI&?aYV#GpHcP#1C$!=aG4c9+mh~o7I3L&tvH>m<&RV5-cZenn zen+1Sd1ekj($Z-I-}3_4vJD&Y*Q#yPPz3#P?K}4iTicEt)u%Ji_hL1w6kq)A*T6c2 zuGn~3I6HV)?9W}ev$L?h5=IG6E}R0lEaLlj2L@`i9`1SXkE?<&V<787LJkj!R1S#} zN<02_O|KQarNjZ!pD6?g2+kjC`VR*D>&pJchOLQmRy~aHA*X?l&|CL;WGl_g2S1ph z+HA3pEb#z={jOEBHn-O-*fJ`V3*@v9Fj`2#V49?-8w@Qq=EgX+tz1ET1h_6f^tI08LU1w9ehFDjN(!LvXeJ5O4D=!0^ zTbh$_KtvP78a=;Wgu}}-ZF({e8 z^?@TAj5KuQDn@3pG@qhWAYe%nS&Y#E6rI72UesTZ3wyB{*4p-P&~ZB9KLsy zBtDLS1IxVz9?;?u4LG9gRm#j2%=w799Y zEa~&~L(pKUEOC{Uw0l8+l2Vs0j8X^dU%&0}>t|00&9|J;$D(o3F~kwa3Qs;%u1^OO z5GW5I7N4uGH;u9t^EU|ULz<2ud;@J$UyspEOjKLnS?9Epf;?iCn}Xm%p`G%%R_SY3 z=>qZ+c9%4RGD3JhCFzGDjBFSaCnOP#q;2EeTlw(0Od@gCuR@RY$(el9=pE3XQWj2{ z-&@Us8q?wK>+_*zHh4{obo*xZvc zlSCRwHaxBc=Tb2lGu$Jz=51ali9%zldk2%B2AN@McIcY7CYbsA?bh^Jb16s&$?lF2 z=^@4&VihV?4P#!gKCxy@bKV7hN>J`byw)QoEdd+f!%}8Sm45)eBkt>J#~~e=;<*a60WXaq_t zRqJiF@AqI5bJT%v2{G(8`|G-skZ)856sjKWZ3!h;geYN{YvGukfsnS>$d_h+dUfiH z(*4DQxVMy1YLhb8j&j;hI_#|(Y7v#s5*qb+7V@|>&W=Y`fz;EQW%%h8!!Hy0^%AMQ zWQ$J&|CmHg$cSxddruiY!*_%fULx(oy;~fwfosX=MD_*YEcBZab^p`{DWN6#C&X`*!~ruw?WAJ2yW63F+p!vf&JHJHIq?!O?p{ zw?lmee`cvz2?c3|aG*7T0p~f9^Yd@+>Kr=uLm>(_N==5q`fqvAnX@^DbpZ?#K1zxg6k#r6mT>(wCQtpg~kFo#R3A>XUa+c=n&SA&j* z6zi!FLoC`^F>ngqb5VDf*wdqD`MVxjc+om;%d`TB)^t2wEB$9M?#&6{I>Y&3nEPAw z^qxkW3ruSXB1Q*u#LI(7qjj0lS)mu^+nG?j7v{Sc#hrb6j-GU&POCx;!_ymz?v>y? z@A>*@zB5-vU@@HxJ`%icES8sNP%s?h#qY=*$~`${r!~3nVZg_yOMdN;rL-gVhLN2@ zm_@Vly)AX?xc6`aLadV?abV_X3qHcFbp`Wg*)TP8=Aqgi`8@sy*JYLgIC`Q{>I~Tg z3^$JUauiY;*^9(^*-P$|9cJ!fgj6mb#r!#?j}5h(r~-WMc=RZgvsYNIaCDt)n&PXdp~g5 zFM#HO#rA?@K=G;B_bxA+bY4MF(zoiaT16NAAgpmzV0^OfxjFhSFQ0T?Nl@~s>aIy8 z;}RT2DcvV4H=?Z~yM1qW@DeTd>gtPaY+m$Q6i|kLe4$`LTQbU{%@db6(sRxseTvgAkK>9QqM94Ws@pMo zHnhG6GRN(biR2Z8o?BPs!>yZzWATGV93MK;CLEGOx?jghh{)zPI8I8Oz%#!bVzA$P z-^^M0fP1YU>0wI`NXnH8dbdtzWyvn{LU2Qbv)U`OH)wiUVg`h=a-)Q#S6FER%_kvc zEuGni^hCRl2=XTalK`<+)0~dB=f`1OX8`0yoIqzbjh$oh?H05`vp2gZMgAjoy9r8z z#&l(`kcKhimSrOD3BXXA8|CE12hGvivs1 z_OM1DoX252XEP|43US_)hGAU6mZN2o{JsQX*=YlHV%~J$Ld0g}Kx~bkByO&m8m(tn z_HtVhz4cQRtWo}S-<2%q6`*Lyp{rwYX;b(q+Xv`LCsNzFl{w5J(M1bth`QlUJ5^uYIRp6MSvN zaZoM~7_V_}k^w!m%-xwpBqp)FIDP(-Mei%2f$qD7ukKi^@UUrQOVP+%4z!C7gAEG3 zjjoJqKfupR4?-JSp_1SC-?TF#H{4!Z-4$4I11IIQDqaLI?nf*!q4khQPGVbgs_{cg z!*UW!@Rq2niQr4eemGkqfoU(H+NSd{J<*u%<@+NC4 zCJn`}T1})8lOEqtBXzx9==iDC{D#)N5hJPJDk*trw;x(dHXBQ&?<>b^Il0kC>M{Fj z;pY41lQA(jjf$4{83sk^4@)2c4Ci9F4qn$h&eyjOG`Rx5d7d>&2r)nqUaWHJ2S@-@ z|2r&oTiwP@htQ zrYtD@C~U`66trjz}wI4N1Z6(kcQQOsW1-onv7$Sh_1I-=MN!3=x@zZJF4 zO(vA$TB642g9(Xi>@?@}UwacdAlXftjWK|SAGn%=K~tz)jzt=OqaR?b=;gyqOnQWn zB<<(*xm96q?$apdaLJ)a9Hr3@(Dii;<-^46A!_r*VHv7Dyr|hTk^+U z%mxOGN%c3P04OaH+ni_SCRsyo%}J;8{VvNXBA7sJuR;UxV%>qu!$<9o`l(N#?^pwp zit4{{A12wx8X+j&&E~dfoT`mrtWFr{)imLk#01ahsQOif5W+XFi@{Y2QSs~cgl3w5 zdV4sZgKnVHGcR+juh{UzN>NnGu`z@`xX#k8Y4Npb_g*yGDtw?&TRhdcQY=F0k+O=c zCJ6^XJMx5?ALKolGMUuoP#?U)1k9Jl2uiN?-X-#B7*x9u)HjBWs#FNlYB!bq{!c}u zAL<#I3zGBc0uiTA$hh*y?XmVybh#0c?yq>=AUxaq>`Tf|t?J@Y$R7Wji3r((~ zaxxoZ@cj6QxoXhy^!?zBV}FL?th=}4;dbO>$MYHYW!2{LYle28^Yi@#%r&3OU4NOitfK82KaKpMeTw_A(0h#tWkCxfY*{41($F5FK>8R2_FO;c50m zw(@pZEi@CJHIYT9R*F|cE}KNUV!3gzm!yq+|8AcpyOMCx6y6d(5odKmFI@(5y~~oL zv8lWeRb{q=@BuZvh_h=-rK4IT62G)XBn&Xm$Gj<975?J{xiTU}L zq}z-rXz_e>)uyti*lL!~+mzG@*bheEu+M=0k$3b;N6K8HOlqb zL_I%HUEG<_2FdJjhhE|cb-==JA>I?{^|HsyIPOg~MPRYS@wtABQsi}7l#L(k^?id# zEEeY$D23@Av%n+DCxWN=g=x9rRekrPIY}erwq;r(McAF$7=CqYWNI5BrgK15gxMl* z{kOW&a26i!i_b}mEm$hfk^NX+4g5};6Gki}yG@fCB!)@&N77|9gJWu!8HS){@g-Js zRE{5EXw0t95^bv9+BpQcV^D^ptQFkih^|QMaRM?_Ju{;b<<{B20EF8S(^T`2dyMfd z9a4u;Fj{XvdSaY->)F%z+SLpg8m#giw?kliTVfMkHC&A%D_I5b8il~X!eg!kguQ;! z70oQwwmwNr|CY{<1dcAQ-iDU29o07;{)OVfZJ59f)CmeQM(6B4kvpBzY**C3=G8SJ zt(E7ytmRJo_%f$C4>c$K*_-G467y+&{z|hktJDka{>!UNAFP#CMUlIcA zBwn5kjmCRqe(kL^K^nqz2V`fSL3idZQj3rkETt@YLjMhw3j3#$zBT+ZvIV^EHo~Zw zHgN7;4wM2-O$Hk1=ZkvX~@8(-SOmQVflX6EWy5^;rW_7cL zSA3mRpdrLkSV-&2J*4MNDFMG~Q&{03oE>(8vex^-j5Cjz{SkuLiyFyq%M-}9lSMro;*g%$>0VTS$2#zmZy@2rAC zU@-Hmwu>O`p>*cW-SBejtU0usKd+~~ALl3KJw;)=kEE1$Q@DeP?1mA`30hGwYpq-q zbkDw2*3c``-{=x?j5?521C?`8>Pv%NvD8N%t8@=ssFBtUYss`m49Xkgobt-Hi7A8cHkC_0CXIH+pm+ehqam0&$K{++L|3OE#RA7``ope zi33wb23Ar;rCeB^HY2}S3jU^ghj1_@%z5#-oh3G?c)Gccp^N{X-_LDrg}mtmLc*&A zx5)J1Mk6b*m{xUYo86_I4{)+96h-`nOjjz68+ws*eqjJf$^x)747YF1}?CN2I;wS2%<=Y=sXOv2dvD)hdN_b_>WwRJj_N@zXo zkt|Mo)p%qd0FUa_a_o4;uD+76Gq=@!l=KSEgsedrB_$$gT0GC0b^jK z8k2ngz)mL6t#6Ow^=T~(-B|AW>+r5=xV0F)Y#{8cG`M4y>^sgThAym7L>$#7HS+Bt z@JWZ{^+_%nYH|jqGGdvux3v^_AYfi?in!Upmy4kah2nCgq1ej3Mu{YZGC2)~ zE`FNsg7{obO~FQ{iKH8iL$)os%rO|S*lO+~pIQ_s+@f%#33ZAid5 zMJ^zzo1VDefomNBH$p%Rw~C^QMRM-BC$EBg!@xa8>cb&QI*{;)SZcb6XuB` zvt`EH&>3*^h0MMq+`eyTQzPD zn|G=306^$SNDtABNtH=h%v+rUS$6P8{nt5sfOZ(;(>LdQ9N8Z2xc#OlV_RC=J8^_% zTlOxPfXN`QAN@*uMr(v2CFF#32xy5c9y15U6Em-nfR51}6)q5f?#^xtwf zJOP)92QUHov%>asF}yQ!G&5CkcC@g4`)f@^s>)d{2_m;)-VuiI@g#4HOkqjoChjCd z;DjnlBbj(u(;bC2*Ek;et!Z&t;8IQC1=U{M?zCwoX+T>UD)`sAB~kA*fjLI4y=y3) z={dMNN&t)AV}IA0FO45QtM+&%GK44Oh_6u+Rjp%=%|WNN-E=w_{`N4ik8+@Dq!BI% z*ET$lZGmKboF!tB>YewgaUne%WZQIy=ba-nI%WyI);wUsbO@h)O1FeDnD6tQ_UWBq zm8NiNU-MVxJ!Jh#ZB&T}cqt^0zi&Mp_IChI3 zh55M_OX>4$X71nhcC{q9O}O7P!bDyOrubbKJx@3utO=uW-f0G!9Z=Q*Ih6bh;q zECRxNJEmbXQSfV}cFu#4fzgvEz4G}(-Yz!%X0(W{s?#7N>lP`;ugrB& zOxK5QMs%@j@Fx*>uY|c^co7v2l18;%aO1m0sHKmw*XWO4HF##70D%2CfIa=5_cN|J zVjS&EGg$Dp3Ff=1?_P^^9YWQd2g~TM=y+p|rs><$z~$wo!v|tDU&B=gE0H~7_4xKK zR7b!19LvA6AQkSxICL9)D>UCSU}cS^hMqY=o>r>ymK6zxOEYn$IOCV4&{ zB*hhL$e$<4H7Is`{@z&V(CsSA$_Ejm$mZ+F#nr2S8-YNIyfy=%=6k?K@y7_*1DA@k znWKuCv-8h2!|>uW6TPv8Ev&hu*`SG}k)k1`(I9mEiQtz=;`g~>#AuZI-GVTTUl^-a zjn<8jvqlCx2RnxtBbI_lAqR~OCyY$Nrisw-uw`M=M4Q24>4-q`jI2#ny!CX!u^@;X zfgSkEEd5y_$Azwan**F7L14KM1$c+4or$8OoxKx-iJha_&uuO;Ue0dmXEg!*ju1Zc zaBz4Yiqyg1b?-AM6l-6|3TwDr-Ece+lgs0P;IA%0c5WV`nbafR)wReGC{Y<~+I zT0(%?dg6Tl@!XMxSGJ##mhgUU?P^zAUF+2Fm zKt#ht>r&7T()@&kb~l*%h5Z{;+Wy1|mrf}$XUSRucCg&EY?{^@)Fs1B%xI5^fTgW& z0}W)EwuM@r5ZvsC?5rQs_XZt88|wrJu*EQ_K4k>(%8NCirIj^N#8q_@BGDgguPZ0m z`42=h;35QZ;y=70dfZb4bPHM1pxN$4d=6>1c~<*wXQaW=oxX2tR_yhq_Mm}B7L+^R zC#lCZChoMRrE3R{Tz-jydoKob!PN$K* z{jXX^tenEH+K}W!DCIR9wj^5Gl``{~?B zK#bjbl)azdUE@Z&1&&{-s6H%oQiLI^8VN@H@q~#dv?9XvC6s;wZh7}r^RzUmV@2!E1)(1W~4uDeNw1?@c`(`eW3g!n_}YPa*<#%_rn`H4U0S!jZPDgFUWMTqj3x8bjvlygWyedX8F#T8ezE zV=Zo`g;q=bixJiwm>lh4^gzp&lN-|g)rt`zR!pLU0PK7bOTAoR2q%f}O%*%ZMil+1 zn}%Jr<_0A?EBf^~Q(F&Dvji0HLp{5kNz8e6;Ya>_KE3oM7z!_B6}zPN$40&Mt1j2| zN*lY3!^6Iv^Yi}dPh-|C%eEB_W7f8zFEUj`EN@Uk@yu?`AWX5d=0+cLWgTyf3|XJh4h6aY=zcTiEss%eg0tQbTUn)< z&O^MJWaa>P&A)n|BF1~(izVSjbIwbsVr9I9IhO@18Y`$igp!)ahm%KXZpeqm50JhU zhuLWf&slEuyqI1Br=YP(B1ZPg5$+EffawVt=V;cH;ffGLInt{NDv8Nb0Pik5eDkgT zjJ(Yr&#kEna~h7huI<+OqzUb9ePbhQRIGw?e~@>H-{csCms3wxR#v~lv_mgJ!71Kh z%#s54F>@%s`?K5D#}kbuXxB=)1AXJA!m+SsCp%WBW5By`WNbNIW(gJM3q0s^X0()w z(a)hab{p%H@!aoNNeKvYZy};6wdi*vQaRFdwFv0_2qk2(j8Ka854E%!)6d(I#{m|C z00%XwvQg#18c3o~Tki?T;J2y7{m9;9-|m(~A8oJXA@flJZp~x%goR>1kIpb2l*^?u zyYci=-mYM_7;p)K6l4&kNvay;eq$}H5Qfv5)D4}f^*)Vspc+97m3vCFc+DzTd3*|F$o=zpYa;Yx zFuG-yaH(%7J=>m0@0HF$E*Z%CySSWAXnPV!+Xp!!kt;}&%nHa+6Qft0D#*$R_)I6E zTE2T?OsG8J@oistWA#j#Jj4N zvWgMSjF1{`rBa79p%{epiBTcBw63WOs|=ZNqu2s+!4{AJt3FRUi+Ze@TgLWi?FkA$ zTODB@H?mQVy}?g3|HC=1M~rK@#@$0516K`Upr+|TUWVW49WI!5#eSq?7;kc>GnI>g zM8FOe)a`Y62$Jzz-aI1z8-*Pwz2GDRX`BzRN?N<_DEn$io@{LW&zI{R`bgS1wGW(J zjAL0Pu;tvY{KV)Xo2@CThhnA6R(Q<8VK`&lnaU1H0o1PETrsIQ1O@f^wrljap;>Das{^w*}>IdYuTqXn5 ztHC>;Sv!*;XE<)tQQ$=>#B0jrHRE)gx8=;=M0thUct$>;8x>HEss@nRu7gXB6fdf9 zTWhat+CIL=@22FeJ4!WEpbZ$Xe+TDS@bUFRh=GYmVX38v%lFtN$^;Zo=f-W2<&vGTAr997ILl`33T+c_N|SB<_@ua$ z3g-I-E8f|tpw2An;>f=;%;d}J8_t&)6Yj~dp)8yyFKsxP z`kUa;!qpBNYAvK9)p__zU%l?UlPst)ZxFDq5Am#`#y;HWMm_?zV_-W0w$qE?Q}ROQ zMoFptr|YawmZ}WsmBwneor=Zru&QcY&l#WV4rheHRrNPwU$Y}d*f}gMO~O&}IML5= zUv<2yd;;ck{^=UnLL80t6&X(a}laKq#6)qaXc^29pU+1B<@XlMUOGc$I zZ=pq%RhnNx5is#v6$W_aG#Cvj6hJ^{wMBg%bL;D2q~aBNv-9h> znLe=z(Fw}CTMhJ&9In&n`~owCcDGZlO0#6Vjcs;%B6z0KyhtaO4%d6pCI-#Qg^G2e zi{9vEdf3ax>f9g*aAvC6D#dxIqtHV{OhtnB^Yz!UL-xWVH)_k6PSZcY%MTCc13z-x zEfDCBfBLB4^-&XnYr;*f`*{vcg6iGK4qqpWx=D#bw>lut_K4!uysY(G(+rCi!=sJU zhAkeTI{#Fdz{aY+7I@`k>Y{1FBud^q^^uwgs}J|9#vERK-X6dLreknfvfp2*^D~9lA$%wZ}eJ zR2PxEkxGe1v>vWd?MbEF<#kDNJ|$*b(ejIRwr2S@CM_mQ3v&11N;9gq@%|EH6dcH&Gr0C}j=^@ZSHwe}cxlK7cX7L#?;0^CT{3j=3Yp?YnDX zw1zjX*Eut@^jWB+QQlo)DA3RzF}=EdDp(QdE;~tCP+^@=WuuIY_9T=57uY?W#XygU zYRja`^QT}wY60Z&ZCDr5o^7L*=83X1h8JYc9le21rjf=PG0nj%OJt7W)^e*OJ#f*p zbAGZByf!Kku_Ge9gPOTbl|FxFM+kJg5j(+P9416rT+yV|?e_kf*f_`@LkZPt=-!Z4 zvJBU3bvfX-^0x*4rxoZXQ;_uhFIMo&;Q!AG{$lt*E3gCx$588Dxu=HC1{qh3(&5uG z*xT&laBVmeQnDr^n$i|)zD1ufpm?N(*BIDBy;5coSiC+!v8y4#=I5^tOm^P>xU2^6 zGd8h%`{CWXKYXZbo29RPy>hbqdAOPq32ut$l)jZK8q~;Z@<=mgMMQP!REf;8mPM4y z`U(LTU$z_{7PWcYLA?%87tezDhYq=(b$o&$I?((Fl-!kW;SJQCQ`ztc>x(esPPmY_ zRys*uLJd*A_mzJ?KJaJ2+ZJR@8Vgi#3E2M0^DIAk-a48G5y*Iul>t);bbAwJ1g>PQE_kyN_|oT@U1)D{cncRwK8{B0P==Z-%Xxt~ zBv8=ML4MS7pP6Iwmu0{2C8p+e{a$BL*QKd9g>@9DYQV(T!ZSO=rCT#f935H0bgW0Jdtrz zqI4i{M=xdlo9oFZmcn|KkjruVvKKQvZ)-$ z_T|G5#3DVkT$u};WbTQmbGwnW}p1 z&6OFx-`pP8&8k6OM+(0l&Z==33EMA#=;t+b^dwVXq&2-5tJc$X>=>6w=ru)4?L+Jji0Ak?M1c5kEY9mqj)3l2HW8c zs8x{n0m*=hu&ocmk_`b7dFaj~SacpJ7@>*FZp!EB)Cxn7skfMavz0$N&+7s#|9|7W zU&iu(oCh4gKYxC6p3yJPbNN*uezzU~^cgLt{njlD>CaG^OH0VutHQS1=8iBTlW~Vr z4or(s%V)m0XO`dB6I|BY7`QaC88))aa$-|%&G>WcIm-E3PxC^D+HH@aBs`$yuA~NU zYUr-PnN@tDBj^)Ej+~#9Q8|U=`*C;Du>nkh{y`a_V6eYRQZPWR{`t_?pQ`@x{x6S! zDa!mE;N_D>zX84i*XCcIJ9;Vj^0}Meg4e*P@5as1Lw3pp@Sv&s?AOH-ofB?TM z=`TfJRvLec%HsYZ`m*l$viC1*Prnho@n4R?zgD7N0=_Ke{08(T_yzc)kn^vFotJvP ztiAjOB_aHmO3X{~m%0Am;%7vE5&uu-|E2WHWcY7sM3R>g;jhs5CBVy&`8NO+^-F-? z0rX3hm!ary6ik|zD8D?`mjEwAyWapXv@ZdEb>qKL{%Y=j51W2tfPkF;4dvg0sGmLh zE7sqg@4sT*(EqhZzp;M1;xDy(>B0R*duIGA*3Yo+*XH&T;H3xk8=#ZvCBU!Y`B#*e zzR7QtGUmUS+8@E@Pfz9LnEZWn{w)pya>Vl2Zu}mT-&?h!3?z^MK|nBozw5wbvKMSW GKm8Y$>}z%a literal 0 HcmV?d00001 From add60aa8705f2481287cf0fe9b77c0aa7b4e78fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:42:05 +0800 Subject: [PATCH 2764/2779] updated plot_uk_marriage_rates.py --- plot_uk_marriage_rates.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py index 6b1297417..a38332443 100755 --- a/plot_uk_marriage_rates.py +++ b/plot_uk_marriage_rates.py @@ -68,10 +68,10 @@ def __init__(self): # super(PlotUKMarriageRates, self).process_options() def run(self): - if not self.args: - self.usage("Provide path to datadownload.xlsx") - - file_path = self.args[0] + file_path = f"{srcdir}/tests/data/uk_marriage_rates_2022.xslx" + if self.args: + file_path = self.args[0] + #self.usage("Provide path to datadownload.xlsx") if not os.path.isfile(file_path): self.usage(f"Invalid argument provided, not a file: {file_path}") From 76d07c239817b67971c1e96aa9246abaf6e7b133 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Nov 2025 23:18:46 +0200 Subject: [PATCH 2765/2779] updated README.md --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 22741ad41..40045f8f9 100644 --- a/README.md +++ b/README.md @@ -637,24 +637,35 @@ Does nothing: [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) -### DBA - SQL +### Databases - DBA - SQL [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) ### DevOps Reloaded -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) + +### Monitoring + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) ### Templates [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) -### Misc +### Desktop + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) + +### Spotify [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) @@ -664,7 +675,4 @@ The rest of my original source repos are Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). - -![](https://hit.yhype.me/github/profile?user_id=2211051) - From 58f3505733d4fedd2abd8baa6bc69a64cde89580 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 04:09:59 -0500 Subject: [PATCH 2766/2779] updated README.md --- README.md | 55 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 40045f8f9..e8cfc70d7 100644 --- a/README.md +++ b/README.md @@ -601,8 +601,8 @@ Hortonworks guy Jonas Straub: ### Knowledge -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) +[![Knowledge-Base](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Diagrams-as-Code](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) ### Containerization -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) +[![Kubernetes-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Dockerfiles](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) ### CI/CD -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) +[![GitHub-Actions](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Jenkins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) ### Databases - DBA - SQL -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) +[![SQL-scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) ### DevOps Reloaded -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) +[![HAProxy-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Terraform](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Packer](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Ansible](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Environments](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) ### Monitoring -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) +[![Nagios-Plugins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Nagios-Plugin-Kafka](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Prometheus](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) ### Templates -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) +[![Templates](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Template-repo](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) ### Desktop -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) ### Spotify -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) +[![Spotify-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Spotify-playlists](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) The rest of my original source repos are [here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). -Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/) +and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo. From ba4689ad10b7936ce3b07791671bb0c2890e2b14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 2767/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index eb10979e3..d65ec80e0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit eb10979e3572ef37b35bda3653b25d7a45e25169 +Subproject commit d65ec80e001bbf9d15f267079c101ae01578a183 From 572228d476a5cbbd8dd6dcf34217b4d5c3e82738 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 2768/2779] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 575080b54..fa8985e60 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 575080b548fec8b2420520e161973a861cbbce30 +Subproject commit fa8985e60aaf5bb26f812cdeeb20b676684e3964 From 8c8d338d2cc4b7ef9282ebe9655a38eb31a4963e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 2769/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2d12574ce..344a446fc 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2d12574ceafbcb13f559dfa46693024616b8e619 +Subproject commit 344a446fc8588cefe7b812c1fc449a484a27c57b From 351e28284fb9a24d1d187c213c5f090ed99dfce7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:12:41 -0500 Subject: [PATCH 2770/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a97248860..22c6dfefe 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a97248860172c6bb3534d08ad38c4cfdfd09c306 +Subproject commit 22c6dfefe61703689e7b09185debd3010d93d9db From 824267aa84016f4cca4b4d6e3b971ef6a91d5a62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:12:41 -0500 Subject: [PATCH 2771/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d65ec80e0..9efc29299 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d65ec80e001bbf9d15f267079c101ae01578a183 +Subproject commit 9efc29299c878e8b781d7714eebcd5c06cef7b7d From 58521d67c5239a3516f9fdb08a35734f46e6ddac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 2772/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 22c6dfefe..8652100df 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 22c6dfefe61703689e7b09185debd3010d93d9db +Subproject commit 8652100dfb9e07f864dff7b9eb3fb144c3640449 From 40f0b93ba154d7da4c0b2c5f565fe89df6bb9f39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 2773/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9efc29299..61ee5978a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9efc29299c878e8b781d7714eebcd5c06cef7b7d +Subproject commit 61ee5978a396f733033af23860014e91969f07e0 From f3cb837c9c2dc10589deaf92c912bccfde093c45 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 2774/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 344a446fc..620b30292 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 344a446fc8588cefe7b812c1fc449a484a27c57b +Subproject commit 620b30292fa572083c77159a1d67f4656b759e1b From 555239dc9cf306ddca0f20cfba69f84de122f431 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 04:56:20 -0300 Subject: [PATCH 2775/2779] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e8cfc70d7..5e1fe9bca 100644 --- a/README.md +++ b/README.md @@ -664,6 +664,7 @@ Does nothing: [![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) [![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![MPV-Scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=MPV-Scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/MPV-Scripts) ### Spotify From a70079597d44a1b0df2cb585dcc3ff96131b73f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:38 -0300 Subject: [PATCH 2776/2779] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8652100df..11dee29ce 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8652100dfb9e07f864dff7b9eb3fb144c3640449 +Subproject commit 11dee29cea607445270d8bd675d1bcdecb069c14 From 511b9e6f20cbc66a10f6d4ed88a45311552f5916 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:38 -0300 Subject: [PATCH 2777/2779] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 61ee5978a..0d272f4cf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 61ee5978a396f733033af23860014e91969f07e0 +Subproject commit 0d272f4cf2c27a8ee614eea18c0bd171e47a06cd From 874083a6f3e5ce387d1496b7509c59b30f05aebb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:39 -0300 Subject: [PATCH 2778/2779] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index fa8985e60..30d80bbd3 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit fa8985e60aaf5bb26f812cdeeb20b676684e3964 +Subproject commit 30d80bbd33f0e6c26c230f26575cb671c41c3e7d From 32ffee202073282e7b7006102e692c453e98b01c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:39 -0300 Subject: [PATCH 2779/2779] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 620b30292..f59532dd6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 620b30292fa572083c77159a1d67f4656b759e1b +Subproject commit f59532dd67a583be3522814a15024a561b12c5ec