From 40680d9a349fe9cc0bde3cdb4d71b7bf5f7da6c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 12:08:31 +0100 Subject: [PATCH 001/333] updated bitbucket-pipelines.yml --- bitbucket-pipelines.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index b63f6a1dc..cdf5243ee 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -13,9 +13,18 @@ # https://www.linkedin.com/in/HariSekhon # -# https://confluence.atlassian.com/x/5Q4SMw -# Only use spaces to indent your .yml configuration. -# ----- +# ============================================================================ # +# B i t b u c k e t C I / C D P i p e l i n e +# ============================================================================ # + +# Reference: +# +# https://support.atlassian.com/bitbucket-cloud/docs/configure-bitbucket-pipelinesyml/ + +# Languages: +# +# https://confluence.atlassian.com/x/5Q4SMw + # You can specify a custom docker image from Docker Hub as your build environment. image: atlassian/default-image:2 From d3f51523fad6fe7dd72db220e7f731389e73f168 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 14:46:10 +0100 Subject: [PATCH 002/333] updated .appveyor.yml --- .appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index d03c2eb3c..f4b36fba0 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -51,6 +51,8 @@ on_finish: # https://github.com/appveyor/ci/pull/3385 # #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/install_openssh.sh' | bash -e - + # + # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi install: From e163110f1502767782682518ac986ae8d67a0ac7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 14:46:11 +0100 Subject: [PATCH 003/333] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 5fdb3401e..6b9eaf3b9 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -52,3 +52,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Validate CODEOWNERS uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From c1adcf6e8e2be7666134ae503ba81bb9f4c20c94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 14:46:11 +0100 Subject: [PATCH 004/333] updated json.yaml --- .github/workflows/json.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index fb73746c5..6ed4b92d1 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -48,3 +48,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Check JSON uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From 145f08ad87e63df6679dbe1ed10281bbbfad6864 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 14:46:12 +0100 Subject: [PATCH 005/333] updated kics.yaml --- .github/workflows/kics.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index b73bedf6f..399cc40f0 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -52,3 +52,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Kics uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From 8a04ebb4873623564d8a97e6704f519465d88b12 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 14:46:13 +0100 Subject: [PATCH 006/333] updated semgrep-cloud.yaml --- .github/workflows/semgrep-cloud.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 9bb52df1a..43b3f5bd2 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -58,3 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} + with: + debug: ${{ github.event.inputs.debug }} From d5f12b600a53504f3ec4559ccf7c1ac65e93c9cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 14:46:13 +0100 Subject: [PATCH 007/333] updated semgrep.yaml --- .github/workflows/semgrep.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index cacc83e09..f1e452f88 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -60,3 +60,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From 3137fc4507d24ecad62f38e99f1f357c69d9839b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 14:46:14 +0100 Subject: [PATCH 008/333] updated yaml.yaml --- .github/workflows/yaml.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index 65d3ead41..bee8be84c 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -54,3 +54,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Check YAML uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From 796cd2e8dca8ccf12eefc085793fa610f9a247d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:08 +0100 Subject: [PATCH 009/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index be1a86ccb..d2862c007 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit be1a86ccb881d89aab8a70786c3b5e0e95f36bac +Subproject commit d2862c007f4bf32388ca16f682991425425e26fc From 3ec3cfc806b30c0164b34a4ab421589492bcf9bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:20 +0100 Subject: [PATCH 010/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1739abe80..d5835bfac 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1739abe80a7bf84d4ca6a7e7834a012473f48159 +Subproject commit d5835bfac3712d68265b8d8ec8a630e55dd124ef From cf9cf126b99287f08a1eda081848212910f05c92 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:23 +0100 Subject: [PATCH 011/333] updated hexanonymize.py --- hexanonymize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hexanonymize.py b/hexanonymize.py index 847d2d73a..b228c50fe 100755 --- a/hexanonymize.py +++ b/hexanonymize.py @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ From 415c28b78b9ea86d4046c154e040f5ee32cfc3ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:23 +0100 Subject: [PATCH 012/333] updated json_to_xml.py --- json_to_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json_to_xml.py b/json_to_xml.py index 0a635c818..32a0a6f80 100755 --- a/json_to_xml.py +++ b/json_to_xml.py @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ From 33df2399751f3c321f778b77d5ffb687bebb1aa3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:24 +0100 Subject: [PATCH 013/333] updated json_to_yaml.py --- json_to_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json_to_yaml.py b/json_to_yaml.py index 172ba2254..b52ce28b7 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ From 8b37bc3f1a6598da9a9390c315ab6e0aebf5a1f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:52 +0100 Subject: [PATCH 014/333] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index fa0a59ca9..ff707a116 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit fa0a59ca990748408802732d50202522b82d52c7 +Subproject commit ff707a1168889fb3ad759c030f3fc633d23d800f From d206a3be37961941aa1aed167bc1d6b81b83120d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:53 +0100 Subject: [PATCH 015/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 27c38c908..86ecc56a8 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 27c38c908387ceacc51463578326ba996449da30 +Subproject commit 86ecc56a8596fcf2fefe577fcaf5898b3fa0d04c From 7466ba5275416a320a7319fec96debc5435d8604 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:54 +0100 Subject: [PATCH 016/333] updated xml_to_json.py --- xml_to_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xml_to_json.py b/xml_to_json.py index bd16d8c46..2b43ce5fe 100755 --- a/xml_to_json.py +++ b/xml_to_json.py @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ From 5405ba99d0bb0e0be5c1a4f9a2bf426f1a78f840 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:55 +0100 Subject: [PATCH 017/333] updated xml_to_yaml.py --- xml_to_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xml_to_yaml.py b/xml_to_yaml.py index 44ad5608f..f947f4a1f 100755 --- a/xml_to_yaml.py +++ b/xml_to_yaml.py @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ From dc4b1ce2b2fbee3797b66501ba3918a900a79769 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 1 Sep 2022 15:32:56 +0100 Subject: [PATCH 018/333] updated yaml_to_json.py --- yaml_to_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yaml_to_json.py b/yaml_to_json.py index 4fa3432cc..e89ae89b9 100755 --- a/yaml_to_json.py +++ b/yaml_to_json.py @@ -12,7 +12,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ From d436d8d95f3171efd52a9082024a82b58a5a1bce Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 12:47:30 +0100 Subject: [PATCH 019/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index d5835bfac..0e86d6fed 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit d5835bfac3712d68265b8d8ec8a630e55dd124ef +Subproject commit 0e86d6fed40a7732d02d39bd0917f7389b6d0549 From 257473a33b959c342be55480deb4e00518ddda26 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 12:47:31 +0100 Subject: [PATCH 020/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d2862c007..64b3cddc4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d2862c007f4bf32388ca16f682991425425e26fc +Subproject commit 64b3cddc4c576b1059834e52512c30c8e3b1c517 From 07d60dc4f1258c9950889776427a4bc982ea29ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 12:47:31 +0100 Subject: [PATCH 021/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 86ecc56a8..193c2afb4 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 86ecc56a8596fcf2fefe577fcaf5898b3fa0d04c +Subproject commit 193c2afb495430cf187af090dc9bdcb8edeea28e From 2f4abcd942af0cf2bae7bfd40d475db0ee342eb8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 13:23:21 +0100 Subject: [PATCH 022/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0e86d6fed..dc7f097f6 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0e86d6fed40a7732d02d39bd0917f7389b6d0549 +Subproject commit dc7f097f630c5d115c6a0309e5245eb055072377 From 61c84bba869c7c9abdf15b8a081597e07ec73ec6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 13:23:58 +0100 Subject: [PATCH 023/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 64b3cddc4..cb7369f50 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 64b3cddc4c576b1059834e52512c30c8e3b1c517 +Subproject commit cb7369f50cf666188ef6d5565d872293fd41b035 From d137353a59af649734b148708ccc7c2096c3f4c5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 13:37:16 +0100 Subject: [PATCH 024/333] updated Makefile --- Makefile | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index c31c2ce74..a67a168a9 100644 --- a/Makefile +++ b/Makefile @@ -86,12 +86,12 @@ init: # @$(MAKE) $@c %.pyc:: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing - @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ echo Generated $@ %.pyo:: %.py - @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile $< && \ echo && \ echo Generated $@ @@ -116,20 +116,20 @@ python: pylib fi @# only install pip packages not installed via system packages - @#$(SUDO_PIP) pip install --upgrade -r requirements.txt - @#$(SUDO_PIP) pip install -r requirements.txt - @PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt + @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt + @#$(SUDO_PIP) $(PIP) install -r requirements.txt + @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit - bash-tools/setup/python_install_snakebite.sh || : + PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi - bash-tools/python_pip_install.sh "avro-python3" || : + @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3"; fi + PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3" || : @# for impyla - @#$(SUDO_PIP) pip install --upgrade setuptools || : + @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || : @# @# snappy may fail to install on Mac not finding snappy-c.h - workaround: @# @@ -140,21 +140,21 @@ python: pylib @# /usr/local/include/snappy-c.h @# @# sudo su - @# LD_RUN_PATH=/usr/local/include pip install snappy + @# LD_RUN_PATH=/usr/local/include $(PIP) install snappy @# - @#$(SUDO_PIP) pip install --upgrade -r requirements.txt + @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt @# for ipython-notebook-pyspark.py - @#$(SUDO_PIP) pip install jinja2 + @#$(SUDO_PIP) PIP=$(PIP) install jinja2 @# HiveServer2 - @#$(SUDO_PIP) pip install pyhs2 + @#$(SUDO_PIP) $(PIP) install pyhs2 @# Impala - @#$(SUDO_PIP) pip install impyla + @#$(SUDO_PIP) $(PIP) install impyla @# must downgrade happybase library to work on Python 2.6 - @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi + @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi @# Python >= 2.7 - won't build on 2.6, handle separately and accept failure - @bash-tools/python_pip_install.sh "ipython[notebook]" || : + @PIP=$(PIP) bash-tools/python_pip_install.sh "ipython[notebook]" || : @echo $(MAKE) pycompile @echo From f7a96e26b072710296b09d82a7f126a8bf948857 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 13:37:31 +0100 Subject: [PATCH 025/333] commented out a bunch of modules that fail to install on new M1 Mac --- requirements.txt | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6060cbe23..9e644234e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,9 @@ GitPython==2.1.15 # this GCP API is surprisingly awful, not using #google-api-python-client==1.11.0 -happybase==1.0.0 +# XXX: install broken on new M1 Mac Python 3.9 +#happybase==1.0.0 + humanize==0.5.1 impyla==0.16.0 jinja2==2.11.3 @@ -36,11 +38,13 @@ ldif3==3.2.2 # Python 3.5+ #numpy==1.17.2 -numpy==1.16.5 +# XXX: install broken on new M1 Mac Python 3.9 +#numpy==1.16.5 # requires pg_config to build from source #psycopg2==2.8.4 -psycopg2-binary==2.8.4 +# XXX: install broken on new M1 Mac Python 3.9 +#psycopg2-binary==2.8.4 python-cson==1.0.9 psutil==5.7.0 @@ -60,8 +64,12 @@ psutil==5.7.0 #python-krbV==1.0.90 # needed by avro -python-snappy==0.5 -sasl==0.2.1 +# XXX: install broken on new M1 Mac Python 3.9 +#python-snappy==0.5 + +# XXX: install broken on new M1 Mac Python 3.9 +#sasl==0.2.1 + sh==1.12.14 selenium==3.141.0 @@ -71,9 +79,11 @@ selenium==3.141.0 #snakebite[kerberos]==2.11.0 #snakebite==2.11.0 -thrift-sasl==0.2.1 -thrift==0.9.3 -thriftpy==0.3.9 +# XXX: install broken on new M1 Mac Python 3.9 +#thrift-sasl==0.2.1 +#thrift==0.9.3 +#thriftpy==0.3.9 + toml==0.10.0 xmltodict==0.10.2 yamllint==1.15.0 From d003d126b3ad855e4c9c35e37bcd127ea890588e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 13:40:01 +0100 Subject: [PATCH 026/333] switched shebang lines to python3 since there is no python2 on new M1 Macs --- ambari_blueprints.py | 2 +- ambari_trigger_service_checks.py | 2 +- anonymize.py | 2 +- aws_s3_presign.py | 2 +- aws_users_access_key_age.py | 2 +- aws_users_last_used.py | 2 +- aws_users_pw_last_used.py | 2 +- aws_users_unused_access_keys.py | 2 +- center.py | 2 +- cloudera_navigator_tables_used.py | 2 +- cloudera_navigator_tables_used_postgres.py | 2 +- crunch_accounting_csv_statement_converter.py | 2 +- docker_registry_show_tags.py | 2 +- dockerfiles_check_git_branches.py | 2 +- dockerfiles_check_git_tags.py | 2 +- dockerhub_search.py | 2 +- dockerhub_show_tags.py | 2 +- find_active_apache_drill.py | 2 +- find_active_apache_drill2.py | 2 +- find_active_cassandra.py | 2 +- find_active_elasticsearch.py | 2 +- find_active_hadoop2_namenode.py | 2 +- find_active_hadoop_namenode.py | 2 +- find_active_hadoop_yarn_resource_manager.py | 2 +- find_active_hbase_master.py | 2 +- find_active_hbase_stargate.py | 2 +- find_active_hbase_thrift.py | 2 +- find_active_impala.py | 2 +- find_active_impala_catalog.py | 2 +- find_active_impala_statestore.py | 2 +- find_active_kubernetes_api.py | 2 +- find_active_oozie.py | 2 +- find_active_presto_coordinator.py | 2 +- find_active_server.py | 2 +- find_active_solrcloud.py | 2 +- find_duplicate_files.py | 2 +- find_missing_files_in_sequence.py | 2 +- gcp_service_account_credential_keys.py | 2 +- getent.py | 2 +- git_check_branches_upstream.py | 2 +- hbase_compact_tables.py | 2 +- hbase_flush_tables.py | 2 +- hbase_generate_data.py | 2 +- hbase_region_requests.py | 2 +- hbase_regions_by_memstore_size.py | 2 +- hbase_regions_by_size.py | 2 +- hbase_regions_least_used.py | 2 +- hbase_regionserver_requests.py | 2 +- hbase_show_table_region_ranges.py | 2 +- hbase_table_region_row_distribution.py | 2 +- hbase_table_row_key_distribution.py | 2 +- hdfs_find_replication_factor_1.py | 2 +- headtail.py | 2 +- hexanonymize.py | 2 +- hive_compute_table_stats.py | 2 +- hive_foreach_table.py | 2 +- hive_schemas_csv.py | 2 +- hive_tables_column_counts.py | 2 +- hive_tables_list.py | 2 +- hive_tables_locations.py | 2 +- hive_tables_metadata.py | 2 +- hive_tables_null_columns.py | 2 +- hive_tables_null_rows.py | 2 +- hive_tables_row_column_counts.py | 2 +- hive_tables_row_counts.py | 2 +- hive_tables_row_counts_any_nulls.py | 2 +- impala_compute_table_stats.py | 2 +- impala_foreach_table.py | 2 +- impala_schemas_csv.py | 2 +- impala_tables_column_counts.py | 2 +- impala_tables_list.py | 2 +- impala_tables_locations.py | 2 +- impala_tables_metadata.py | 2 +- impala_tables_null_columns.py | 2 +- impala_tables_null_rows.py | 2 +- impala_tables_row_column_counts.py | 2 +- impala_tables_row_counts.py | 2 +- impala_tables_row_counts_any_nulls.py | 2 +- ipython_notebook_pyspark.py | 2 +- json_docs_to_bulk_multiline.py | 2 +- json_to_xml.py | 2 +- json_to_yaml.py | 2 +- opentsdb_import_metric_distribution.py | 2 +- python_find_library_path.py | 2 +- pythonpath.py | 2 +- quay_show_tags.py | 2 +- selenium_hub_browser_test.py | 2 +- serf_event_handler.py | 2 +- spark_avro_to_parquet.py | 2 +- spark_csv_to_avro.py | 2 +- spark_csv_to_parquet.py | 2 +- spark_json_to_avro.py | 2 +- spark_json_to_parquet.py | 2 +- spark_parquet_to_avro.py | 2 +- strip_ansi_escape_codes.py | 2 +- timeout.py | 2 +- travis_debug_session.py | 2 +- travis_last_log.py | 2 +- urldecode.py | 2 +- urlencode.py | 2 +- validate_avro.py | 2 +- validate_cson.py | 2 +- validate_csv.py | 2 +- validate_ini.py | 2 +- validate_ini2.py | 2 +- validate_json.py | 2 +- validate_ldap_ldif.py | 2 +- validate_multimedia.py | 2 +- validate_parquet.py | 2 +- validate_toml.py | 2 +- validate_xml.py | 2 +- validate_yaml.py | 2 +- welcome.py | 2 +- xml_to_json.py | 2 +- xml_to_yaml.py | 2 +- yaml_to_json.py | 2 +- 116 files changed, 116 insertions(+), 116 deletions(-) diff --git a/ambari_blueprints.py b/ambari_blueprints.py index 8e36ba289..e5dd52217 100755 --- a/ambari_blueprints.py +++ b/ambari_blueprints.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/ambari_trigger_service_checks.py b/ambari_trigger_service_checks.py index 8a2247432..2afe64207 100755 --- a/ambari_trigger_service_checks.py +++ b/ambari_trigger_service_checks.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/anonymize.py b/anonymize.py index ee3f023f1..1d85311f1 100755 --- a/anonymize.py +++ b/anonymize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/aws_s3_presign.py b/aws_s3_presign.py index ec8b6d444..e1addfbbc 100755 --- a/aws_s3_presign.py +++ b/aws_s3_presign.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 748a248e3..be218c2db 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 903222117..e06a7bcdb 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py index 72c922145..6810abd17 100755 --- a/aws_users_pw_last_used.py +++ b/aws_users_pw_last_used.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py index 58cb30389..befb152ee 100755 --- a/aws_users_unused_access_keys.py +++ b/aws_users_unused_access_keys.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/center.py b/center.py index 80b17ba0e..4856ea0f7 100755 --- a/center.py +++ b/center.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # args: -s <<< "Auth & Config" # args: -s <<< "GKE Clusters" diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 426ad4c4a..bbd94dbb3 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py index 324985959..972716236 100755 --- a/cloudera_navigator_tables_used_postgres.py +++ b/cloudera_navigator_tables_used_postgres.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index ac84f3141..2e675c253 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/docker_registry_show_tags.py b/docker_registry_show_tags.py index c2b29bde1..8c18d3cdc 100755 --- a/docker_registry_show_tags.py +++ b/docker_registry_show_tags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/dockerfiles_check_git_branches.py b/dockerfiles_check_git_branches.py index 444d49ea1..662dce3e2 100755 --- a/dockerfiles_check_git_branches.py +++ b/dockerfiles_check_git_branches.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/dockerfiles_check_git_tags.py b/dockerfiles_check_git_tags.py index 62ec4f361..3e0b627bd 100755 --- a/dockerfiles_check_git_tags.py +++ b/dockerfiles_check_git_tags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/dockerhub_search.py b/dockerhub_search.py index 24f33c692..ba7034145 100755 --- a/dockerhub_search.py +++ b/dockerhub_search.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # args: harisekhon # diff --git a/dockerhub_show_tags.py b/dockerhub_show_tags.py index fafc985f2..59cd6cb5f 100755 --- a/dockerhub_show_tags.py +++ b/dockerhub_show_tags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_apache_drill.py b/find_active_apache_drill.py index ece0d22bf..1e89fb676 100755 --- a/find_active_apache_drill.py +++ b/find_active_apache_drill.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_apache_drill2.py b/find_active_apache_drill2.py index ec1557923..364b0eda3 100755 --- a/find_active_apache_drill2.py +++ b/find_active_apache_drill2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_cassandra.py b/find_active_cassandra.py index 2baa09140..c16ee217c 100755 --- a/find_active_cassandra.py +++ b/find_active_cassandra.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_elasticsearch.py b/find_active_elasticsearch.py index d878f5e42..be449bc79 100755 --- a/find_active_elasticsearch.py +++ b/find_active_elasticsearch.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hadoop2_namenode.py b/find_active_hadoop2_namenode.py index d041b517a..7aad7e77d 100755 --- a/find_active_hadoop2_namenode.py +++ b/find_active_hadoop2_namenode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hadoop_namenode.py b/find_active_hadoop_namenode.py index 0a058fa2b..1f6f2d44d 100755 --- a/find_active_hadoop_namenode.py +++ b/find_active_hadoop_namenode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hadoop_yarn_resource_manager.py b/find_active_hadoop_yarn_resource_manager.py index de6967c4c..f2f0e48e3 100755 --- a/find_active_hadoop_yarn_resource_manager.py +++ b/find_active_hadoop_yarn_resource_manager.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hbase_master.py b/find_active_hbase_master.py index e9edd2a7f..cc7de0222 100755 --- a/find_active_hbase_master.py +++ b/find_active_hbase_master.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hbase_stargate.py b/find_active_hbase_stargate.py index 89a417f84..79dc122fa 100755 --- a/find_active_hbase_stargate.py +++ b/find_active_hbase_stargate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hbase_thrift.py b/find_active_hbase_thrift.py index 88ea248a7..06e44a414 100755 --- a/find_active_hbase_thrift.py +++ b/find_active_hbase_thrift.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_impala.py b/find_active_impala.py index 4fcc6470a..ae086f46a 100755 --- a/find_active_impala.py +++ b/find_active_impala.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_impala_catalog.py b/find_active_impala_catalog.py index 4b4bdb248..cb6ea0de3 100755 --- a/find_active_impala_catalog.py +++ b/find_active_impala_catalog.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_impala_statestore.py b/find_active_impala_statestore.py index 4edf2c947..049ed78f5 100755 --- a/find_active_impala_statestore.py +++ b/find_active_impala_statestore.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_kubernetes_api.py b/find_active_kubernetes_api.py index 54bbf93d6..a93b0bc81 100755 --- a/find_active_kubernetes_api.py +++ b/find_active_kubernetes_api.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_oozie.py b/find_active_oozie.py index ae23d74fd..156bb51a9 100755 --- a/find_active_oozie.py +++ b/find_active_oozie.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_presto_coordinator.py b/find_active_presto_coordinator.py index 679ec19ae..9b0050a47 100755 --- a/find_active_presto_coordinator.py +++ b/find_active_presto_coordinator.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_server.py b/find_active_server.py index ef2b3e7c2..30810a448 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_solrcloud.py b/find_active_solrcloud.py index 0605454b3..659eceb59 100755 --- a/find_active_solrcloud.py +++ b/find_active_solrcloud.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 5735625bd..4c452b03f 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index d6c07c85b..f982cbddd 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py index 757ebe3f8..f58f86e98 100755 --- a/gcp_service_account_credential_keys.py +++ b/gcp_service_account_credential_keys.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/getent.py b/getent.py index 90f19a4a2..d8b701cd6 100755 --- a/getent.py +++ b/getent.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/git_check_branches_upstream.py b/git_check_branches_upstream.py index f367ab553..27ec09865 100755 --- a/git_check_branches_upstream.py +++ b/git_check_branches_upstream.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_compact_tables.py b/hbase_compact_tables.py index e52db1a69..7bfca40b2 100755 --- a/hbase_compact_tables.py +++ b/hbase_compact_tables.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_flush_tables.py b/hbase_flush_tables.py index 9fd63ec1f..71be25c17 100755 --- a/hbase_flush_tables.py +++ b/hbase_flush_tables.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_generate_data.py b/hbase_generate_data.py index 33e84d38b..3ed31a644 100755 --- a/hbase_generate_data.py +++ b/hbase_generate_data.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_region_requests.py b/hbase_region_requests.py index 24d60bc6e..d182431c8 100755 --- a/hbase_region_requests.py +++ b/hbase_region_requests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_regions_by_memstore_size.py b/hbase_regions_by_memstore_size.py index 14934fca6..295653a81 100755 --- a/hbase_regions_by_memstore_size.py +++ b/hbase_regions_by_memstore_size.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_regions_by_size.py b/hbase_regions_by_size.py index ea4bb5b75..c67184c8b 100755 --- a/hbase_regions_by_size.py +++ b/hbase_regions_by_size.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_regions_least_used.py b/hbase_regions_least_used.py index 8f13bda7c..752be4ed5 100755 --- a/hbase_regions_least_used.py +++ b/hbase_regions_least_used.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_regionserver_requests.py b/hbase_regionserver_requests.py index 7c2638407..a8c96ebb9 100755 --- a/hbase_regionserver_requests.py +++ b/hbase_regionserver_requests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_show_table_region_ranges.py b/hbase_show_table_region_ranges.py index bb2ff7f9c..585e11cf9 100755 --- a/hbase_show_table_region_ranges.py +++ b/hbase_show_table_region_ranges.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_table_region_row_distribution.py b/hbase_table_region_row_distribution.py index 0c7dd702b..0dfe6e024 100755 --- a/hbase_table_region_row_distribution.py +++ b/hbase_table_region_row_distribution.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_table_row_key_distribution.py b/hbase_table_row_key_distribution.py index aeebf0e27..6059763d7 100755 --- a/hbase_table_row_key_distribution.py +++ b/hbase_table_row_key_distribution.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py index 8745848be..1f348556d 100755 --- a/hdfs_find_replication_factor_1.py +++ b/hdfs_find_replication_factor_1.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/headtail.py b/headtail.py index d519608da..ec21a3f5c 100755 --- a/headtail.py +++ b/headtail.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hexanonymize.py b/hexanonymize.py index b228c50fe..cac35e6a0 100755 --- a/hexanonymize.py +++ b/hexanonymize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py index c1bf4073d..d416f9379 100755 --- a/hive_compute_table_stats.py +++ b/hive_compute_table_stats.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 7fb735421..9d5abf6d6 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index da1d019c4..ea41f3753 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py index 5891882be..d6cb4ce68 100755 --- a/hive_tables_column_counts.py +++ b/hive_tables_column_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_list.py b/hive_tables_list.py index 6b1cc0197..64675c61f 100755 --- a/hive_tables_list.py +++ b/hive_tables_list.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_locations.py b/hive_tables_locations.py index 69f62c0f7..403e07443 100755 --- a/hive_tables_locations.py +++ b/hive_tables_locations.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 884715623..d15d62feb 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 81313be6b..207fcba3f 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 17e85f69a..f1a5752f1 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_row_column_counts.py b/hive_tables_row_column_counts.py index 2185eac7a..c2a5ec7e9 100755 --- a/hive_tables_row_column_counts.py +++ b/hive_tables_row_column_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index b36b46c7c..27aaa1a9c 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 21b592dcf..8ea7e6b75 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py index 95c185b46..04d462e16 100755 --- a/impala_compute_table_stats.py +++ b/impala_compute_table_stats.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 495d77f25..f4e324ac9 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py index 8575c88f9..0ac8491dd 100755 --- a/impala_schemas_csv.py +++ b/impala_schemas_csv.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py index ddfadc37a..e62027542 100755 --- a/impala_tables_column_counts.py +++ b/impala_tables_column_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_list.py b/impala_tables_list.py index b9dd00d5d..a45194ad3 100755 --- a/impala_tables_list.py +++ b/impala_tables_list.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_locations.py b/impala_tables_locations.py index 2203f6caf..383bb1154 100755 --- a/impala_tables_locations.py +++ b/impala_tables_locations.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 50ff43fc2..501b82c93 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index 948ba2e8e..2b70caf3f 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py index 2e28d9a4e..4fea8a82e 100755 --- a/impala_tables_null_rows.py +++ b/impala_tables_null_rows.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_row_column_counts.py b/impala_tables_row_column_counts.py index 907d8f321..8cf4d6b82 100755 --- a/impala_tables_row_column_counts.py +++ b/impala_tables_row_column_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 9c968a330..decc2982c 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py index e7fa552d0..d7b49a3b2 100755 --- a/impala_tables_row_counts_any_nulls.py +++ b/impala_tables_row_counts_any_nulls.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/ipython_notebook_pyspark.py b/ipython_notebook_pyspark.py index 7f0c61967..4c4505015 100755 --- a/ipython_notebook_pyspark.py +++ b/ipython_notebook_pyspark.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Author: Hari Sekhon # Date: 6/8/2014 diff --git a/json_docs_to_bulk_multiline.py b/json_docs_to_bulk_multiline.py index 6cb0c21cc..eb49bcbeb 100755 --- a/json_docs_to_bulk_multiline.py +++ b/json_docs_to_bulk_multiline.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/json_to_xml.py b/json_to_xml.py index 32a0a6f80..54c0d6395 100755 --- a/json_to_xml.py +++ b/json_to_xml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/json_to_yaml.py b/json_to_yaml.py index b52ce28b7..674f6729f 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/opentsdb_import_metric_distribution.py b/opentsdb_import_metric_distribution.py index eb303b545..3b3d3df09 100755 --- a/opentsdb_import_metric_distribution.py +++ b/opentsdb_import_metric_distribution.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/python_find_library_path.py b/python_find_library_path.py index 92417b56e..66fb9c00e 100755 --- a/python_find_library_path.py +++ b/python_find_library_path.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/pythonpath.py b/pythonpath.py index d0dde835c..ef42891b9 100755 --- a/pythonpath.py +++ b/pythonpath.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/quay_show_tags.py b/quay_show_tags.py index 716221ece..d1eef86bf 100755 --- a/quay_show_tags.py +++ b/quay_show_tags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index f302a3f56..c3d117e53 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/serf_event_handler.py b/serf_event_handler.py index 3d1ad7afe..455d7bee1 100755 --- a/serf_event_handler.py +++ b/serf_event_handler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py index e1befc15f..9300c4fd1 100755 --- a/spark_avro_to_parquet.py +++ b/spark_avro_to_parquet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index 1844e487a..fb56e18cf 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py index 90ea7355b..8e3b0da1c 100755 --- a/spark_csv_to_parquet.py +++ b/spark_csv_to_parquet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index 6dab95cc5..2922c7f7f 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py index f6fa7e7bf..2ed935917 100755 --- a/spark_json_to_parquet.py +++ b/spark_json_to_parquet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py index cd026dd19..0186981f5 100755 --- a/spark_parquet_to_avro.py +++ b/spark_parquet_to_avro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/strip_ansi_escape_codes.py b/strip_ansi_escape_codes.py index 021f59b00..24738194a 100755 --- a/strip_ansi_escape_codes.py +++ b/strip_ansi_escape_codes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/timeout.py b/timeout.py index 6e33334ec..831d2e99f 100755 --- a/timeout.py +++ b/timeout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/travis_debug_session.py b/travis_debug_session.py index 1c487b221..aafebf35e 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/travis_last_log.py b/travis_last_log.py index 2aa861cff..82b7493e5 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/urldecode.py b/urldecode.py index 7356b84db..87319d3be 100755 --- a/urldecode.py +++ b/urldecode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/urlencode.py b/urlencode.py index f494f0d3f..6ca907f15 100755 --- a/urlencode.py +++ b/urlencode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_avro.py b/validate_avro.py index 718d05690..37eaab848 100755 --- a/validate_avro.py +++ b/validate_avro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_cson.py b/validate_cson.py index a64ccbc78..6e017ab97 100755 --- a/validate_cson.py +++ b/validate_cson.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_csv.py b/validate_csv.py index 2e7d2f78c..bac1d583e 100755 --- a/validate_csv.py +++ b/validate_csv.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_ini.py b/validate_ini.py index 6d731c864..54a01cc63 100755 --- a/validate_ini.py +++ b/validate_ini.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_ini2.py b/validate_ini2.py index a987a9474..5c41436a4 100755 --- a/validate_ini2.py +++ b/validate_ini2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_json.py b/validate_json.py index 3ffa53821..dbca2c431 100755 --- a/validate_json.py +++ b/validate_json.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_ldap_ldif.py b/validate_ldap_ldif.py index d27f333d7..963b725c8 100755 --- a/validate_ldap_ldif.py +++ b/validate_ldap_ldif.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/validate_multimedia.py b/validate_multimedia.py index 635833c3f..0362bb30d 100755 --- a/validate_multimedia.py +++ b/validate_multimedia.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_parquet.py b/validate_parquet.py index c707d6c90..bf6690d06 100755 --- a/validate_parquet.py +++ b/validate_parquet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_toml.py b/validate_toml.py index 96e029619..25628d222 100755 --- a/validate_toml.py +++ b/validate_toml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_xml.py b/validate_xml.py index 10b73b438..c954bac42 100755 --- a/validate_xml.py +++ b/validate_xml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_yaml.py b/validate_yaml.py index ebfb11ef8..75ee4034c 100755 --- a/validate_yaml.py +++ b/validate_yaml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/welcome.py b/welcome.py index d9246e6b4..d9c1b2546 100755 --- a/welcome.py +++ b/welcome.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/xml_to_json.py b/xml_to_json.py index 2b43ce5fe..14d4cde62 100755 --- a/xml_to_json.py +++ b/xml_to_json.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/xml_to_yaml.py b/xml_to_yaml.py index f947f4a1f..47a098243 100755 --- a/xml_to_yaml.py +++ b/xml_to_yaml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/yaml_to_json.py b/yaml_to_json.py index e89ae89b9..90a5db9b7 100755 --- a/yaml_to_json.py +++ b/yaml_to_json.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # args: ../bash-tools/.gitlab-ci.yml # From 8b053e6d64aeba718fd57414f5e493b22795a09d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 13:42:12 +0100 Subject: [PATCH 027/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index cb7369f50..c5af6c602 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit cb7369f50cf666188ef6d5565d872293fd41b035 +Subproject commit c5af6c60232af834fe34a3123f2e58e9dc4749b0 From 21637c2f5eb3d257e39806967061d7f47eaeed8c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 13:52:51 +0100 Subject: [PATCH 028/333] updated buddy.yml --- buddy.yml | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/buddy.yml b/buddy.yml index 82a6a21ef..471b5da33 100644 --- a/buddy.yml +++ b/buddy.yml @@ -15,6 +15,7 @@ # https://buddy.works/docs/yaml/yaml-schema +--- - pipeline: "Build" trigger_mode: "ON_EVERY_PUSH" ref_name: "master" @@ -22,22 +23,22 @@ target_site_url: "https://github.com/HariSekhon/DevOps-Python-tools" trigger_condition: "ALWAYS" actions: - - action: "Execute: make ci test" - type: "BUILD" - working_directory: "/buddy/devops-python-tools" - docker_image_name: "library/ubuntu" - docker_image_tag: "18.04" - #setup_commands: - # this step gets cached, which results in - # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? - # - apt update - # - apt install -qy git make - execute_commands: - - setup/ci_bootstrap.sh - - make init - - make ci - - make test - volume_mappings: - - "/:/buddy/devops-python-tools" - shell: "BASH" - trigger_condition: "ALWAYS" + - action: "Execute: make ci test" + type: "BUILD" + working_directory: "/buddy/devops-python-tools" + docker_image_name: "library/ubuntu" + docker_image_tag: "18.04" + #setup_commands: + # this step gets cached, which results in + # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? + # - apt update + # - apt install -qy git make + execute_commands: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test + volume_mappings: + - "/:/buddy/devops-python-tools" + shell: "BASH" + trigger_condition: "ALWAYS" From 773b7aaebe78d2c9099c43341842bfc2bb92f7cd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 13:52:57 +0100 Subject: [PATCH 029/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c5af6c602..779f5d7f0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c5af6c60232af834fe34a3123f2e58e9dc4749b0 +Subproject commit 779f5d7f0fb60488691b94abef67a21070ec2772 From e1b1fa48ff364133ae1f1adf4e953937f904b518 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 15:39:00 +0100 Subject: [PATCH 030/333] updatd codeship.yml --- codeship.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codeship.yml b/codeship.yml index a65064472..f466bd6a7 100644 --- a/codeship.yml +++ b/codeship.yml @@ -17,6 +17,8 @@ # C o d e S h i p # ============================================================================ # +# LEGACY - CloudBees has removed free plans as of 1st March 2023 + # 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this # https://github.com/painless-software/codeship-yaml From 5cf497b0fac1e17efbe3566a39a3f4e61b370b89 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 15:39:51 +0100 Subject: [PATCH 031/333] updated gocd config --- setup/gocd_config_repo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json index 5071aa87d..4c1639b4d 100644 --- a/setup/gocd_config_repo.json +++ b/setup/gocd_config_repo.json @@ -1,5 +1,5 @@ { - "id": "devops-python-tools", + "id": "DevOps-Python-tools", "plugin_id": "yaml.config.plugin", "material": { "type": "git", From 449c6f949314038338dc9ee998f198d1c9ae7416 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 15:49:06 +0100 Subject: [PATCH 032/333] updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index b313668af..917e3b8e0 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,6 @@ [![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) [![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) [![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) -[![Codeship Status for HariSekhon/DevOps-Python-tools](https://app.codeship.com/projects/b281baa0-3c5f-0138-caef-66210e546d42/status?branch=master)](https://app.codeship.com/projects/387251) [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) From e67e38c4069bd6876b8a5f36a6447e54de810571 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 15:52:52 +0100 Subject: [PATCH 033/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index dc7f097f6..0caddf7a5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit dc7f097f630c5d115c6a0309e5245eb055072377 +Subproject commit 0caddf7a58297dd75d19ecfea71eb2206853d4c4 From c56e8fbc26c7b238a3f64abe1286a5868adad458 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 15:52:52 +0100 Subject: [PATCH 034/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 779f5d7f0..357fcd97c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 779f5d7f0fb60488691b94abef67a21070ec2772 +Subproject commit 357fcd97c3ccf3b6f89aa07ef71c994504e7c7c8 From 26b211ef89f99e006abd8782cbe8da9aec0479a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 13 Apr 2023 15:52:52 +0100 Subject: [PATCH 035/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 193c2afb4..7373f570f 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 193c2afb495430cf187af090dc9bdcb8edeea28e +Subproject commit 7373f570ffb1a790ec576ef4c6cf52dcf0785ba0 From 80c142507b022273b11a7059c1937231878ee83b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Apr 2023 23:40:20 +0100 Subject: [PATCH 036/333] updated azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b92c0d211..76ad8f1f0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,7 @@ pool: # there is no /dev/stderr on this azure build! #vmImage: 'ubuntu-latest' # Ubuntu 16.04 required for docker container support, looks like 18.04 works too - vmImage: 'ubuntu-18.04' + vmImage: 'ubuntu-22.04' # unprivileged container without sudo, cannot install dependencies #container: ubuntu:18.04 From b46fccee299f599876f845fa2ba8bf04382b1c0f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Apr 2023 15:22:56 +0100 Subject: [PATCH 037/333] updated .gocd.yml --- .gocd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gocd.yml b/.gocd.yml index d3a89a0ef..f3ed9fd19 100644 --- a/.gocd.yml +++ b/.gocd.yml @@ -19,7 +19,7 @@ --- format_version: 3 pipelines: - devops-python-tools: + DevOps-Python-tools: group: defaultGroup label_template: ${COUNT} lock_behavior: none From 22c537d032d483f36d7c211896e0bf38a43c4d09 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Apr 2023 15:23:26 +0100 Subject: [PATCH 038/333] updated buddy.yml --- buddy.yml | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/buddy.yml b/buddy.yml index 471b5da33..82fec7993 100644 --- a/buddy.yml +++ b/buddy.yml @@ -15,7 +15,6 @@ # https://buddy.works/docs/yaml/yaml-schema ---- - pipeline: "Build" trigger_mode: "ON_EVERY_PUSH" ref_name: "master" @@ -23,22 +22,22 @@ target_site_url: "https://github.com/HariSekhon/DevOps-Python-tools" trigger_condition: "ALWAYS" actions: - - action: "Execute: make ci test" - type: "BUILD" - working_directory: "/buddy/devops-python-tools" - docker_image_name: "library/ubuntu" - docker_image_tag: "18.04" - #setup_commands: - # this step gets cached, which results in - # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? - # - apt update - # - apt install -qy git make - execute_commands: - - setup/ci_bootstrap.sh - - make init - - make ci - - make test - volume_mappings: - - "/:/buddy/devops-python-tools" - shell: "BASH" - trigger_condition: "ALWAYS" + - action: "Execute: make ci test" + type: "BUILD" + working_directory: "/buddy/DevOps-Python-tools" + docker_image_name: "library/ubuntu" + docker_image_tag: "18.04" + #setup_commands: + # this step gets cached, which results in + # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? + # - apt update + # - apt install -qy git make + execute_commands: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test + volume_mappings: + - "/:/buddy/DevOps-Python-tools" + shell: "BASH" + trigger_condition: "ALWAYS" From 29a2b9950658e2c20ddc88ad0e069ecb3b5de2e6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Apr 2023 15:24:49 +0100 Subject: [PATCH 039/333] updated buddy.yml --- buddy.yml | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/buddy.yml b/buddy.yml index 82fec7993..f901a6425 100644 --- a/buddy.yml +++ b/buddy.yml @@ -15,6 +15,7 @@ # https://buddy.works/docs/yaml/yaml-schema +--- - pipeline: "Build" trigger_mode: "ON_EVERY_PUSH" ref_name: "master" @@ -22,22 +23,22 @@ target_site_url: "https://github.com/HariSekhon/DevOps-Python-tools" trigger_condition: "ALWAYS" actions: - - action: "Execute: make ci test" - type: "BUILD" - working_directory: "/buddy/DevOps-Python-tools" - docker_image_name: "library/ubuntu" - docker_image_tag: "18.04" - #setup_commands: - # this step gets cached, which results in - # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? - # - apt update - # - apt install -qy git make - execute_commands: - - setup/ci_bootstrap.sh - - make init - - make ci - - make test - volume_mappings: - - "/:/buddy/DevOps-Python-tools" - shell: "BASH" - trigger_condition: "ALWAYS" + - action: "Execute: make ci test" + type: "BUILD" + working_directory: "/buddy/DevOps-Python-tools" + docker_image_name: "library/ubuntu" + docker_image_tag: "18.04" + #setup_commands: + # this step gets cached, which results in + # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? + # - apt update + # - apt install -qy git make + execute_commands: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test + volume_mappings: + - "/:/buddy/DevOps-Python-tools" + shell: "BASH" + trigger_condition: "ALWAYS" From bdd4eec6c099b4c2c21fd87cdff60c7383cc0de9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Apr 2023 23:56:22 +0100 Subject: [PATCH 040/333] updated README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 917e3b8e0..152556544 100644 --- a/README.md +++ b/README.md @@ -501,6 +501,8 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Perl Lib](https://github.com/HariSekhon/lib) - Perl version of above library +- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, ArgoCD, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... + From bccc01b7473cf51d3a5be02a09f379fe3ff569fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:42 +0100 Subject: [PATCH 041/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0caddf7a5..5e7dfbdc8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0caddf7a58297dd75d19ecfea71eb2206853d4c4 +Subproject commit 5e7dfbdc8cea422b6856153eea3c01df4c84a60c From 5a0a33122adaa05720ea69222186feae21887c6f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:42 +0100 Subject: [PATCH 042/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 357fcd97c..4515bb958 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 357fcd97c3ccf3b6f89aa07ef71c994504e7c7c8 +Subproject commit 4515bb958f4b27c7c5854ba77ee665b41852dd5b From cf3544f48c0168b9be909b530a3c071d9b46aad7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:43 +0100 Subject: [PATCH 043/333] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ff707a116..d4eedc0e2 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ff707a1168889fb3ad759c030f3fc633d23d800f +Subproject commit d4eedc0e2c68c9a43838c89a1d09abf10927a566 From 194ca3daa90416a88ee399a7b9e7c3f0d7bd90c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:43 +0100 Subject: [PATCH 044/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 7373f570f..8d838daa6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 7373f570ffb1a790ec576ef4c6cf52dcf0785ba0 +Subproject commit 8d838daa669dc05d497d5fb2fa1aef631c613acf From 2b13a79338d74de5b5ea78f59cd02f79484e1c61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:28:26 +0100 Subject: [PATCH 045/333] renamed debian_8.yaml to debian_8.yaml.disabled --- .github/workflows/{debian_8.yaml => debian_8.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_8.yaml => debian_8.yaml.disabled} (100%) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml.disabled similarity index 100% rename from .github/workflows/debian_8.yaml rename to .github/workflows/debian_8.yaml.disabled From 8ef52b84f1c5e3aa90439b660ec903ee8c132e05 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:37:40 +0100 Subject: [PATCH 046/333] added debian_11.yaml --- .github/workflows/debian_11.yaml | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/debian_11.yaml diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml new file mode 100644 index 000000000..de3b4b96f --- /dev/null +++ b/.github/workflows/debian_11.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 11 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_11.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:11 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} From ed38e3efe36f2d5fc3eae3f90b6f50d82d538a47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:49:50 +0100 Subject: [PATCH 047/333] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 152556544..af9216972 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ [![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) -[![Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+8%22) [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) +[![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) [![CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22) [![CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22) [![CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22) From 5ba4480dc89627d3c39d1f8de03f3450d709fa7c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:42:16 +0100 Subject: [PATCH 048/333] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index af9216972..c6ae3495f 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,9 @@ [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) -[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml) [![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) - [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) From 824e98dd0d38a3382c69dac5eb7e85b1056836ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:49:30 +0100 Subject: [PATCH 049/333] renamed python3.6.yaml to python3.6.yaml.disabled --- .github/workflows/{python3.6.yaml => python3.6.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python3.6.yaml => python3.6.yaml.disabled} (100%) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml.disabled similarity index 100% rename from .github/workflows/python3.6.yaml rename to .github/workflows/python3.6.yaml.disabled From 7408f3ce0898dea20692929e949e336edd98dfe8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:49:36 +0100 Subject: [PATCH 050/333] renamed python2.7.yaml to python2.7.yaml.disabled --- .github/workflows/{python2.7.yaml => python2.7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python2.7.yaml => python2.7.yaml.disabled} (100%) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml.disabled similarity index 100% rename from .github/workflows/python2.7.yaml rename to .github/workflows/python2.7.yaml.disabled From 83b8b0506a21b3636ba4f75627f965c01fb0545f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:51:59 +0100 Subject: [PATCH 051/333] added python3.11.yaml --- .github/workflows/python3.11.yaml | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/python3.11.yaml diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml new file mode 100644 index 000000000..8e2d041ac --- /dev/null +++ b/.github/workflows/python3.11.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.11 + +on: + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.11 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: "3.11" + caches: apt pip + debug: ${{ github.event.inputs.debug }} From 1cfb67773c00608d7fe6fc109f46a968747f63bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:53:20 +0100 Subject: [PATCH 052/333] updated README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index c6ae3495f..5b9392d62 100644 --- a/README.md +++ b/README.md @@ -101,12 +101,11 @@ [![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) [![Python versions](https://img.shields.io/badge/Python-2.7+-3776AB?logo=python&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools) -[![Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+2.7%22) -[![Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.6%22) [![Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22) [![Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22) [![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) +[![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) [![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) [![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) From d824e50952a1bb22fef5061544e37305f6b37623 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:56:31 +0100 Subject: [PATCH 053/333] renamed pypy2.yaml to pypy2.yaml.disabled --- .github/workflows/{pypy2.yaml => pypy2.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pypy2.yaml => pypy2.yaml.disabled} (100%) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml.disabled similarity index 100% rename from .github/workflows/pypy2.yaml rename to .github/workflows/pypy2.yaml.disabled From 9091c212bd734a1eb605f29d3ecf516fcb12cbcb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:56:31 +0100 Subject: [PATCH 054/333] renamed pypy3.yaml to pypy3.yaml.disabled --- .github/workflows/{pypy3.yaml => pypy3.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pypy3.yaml => pypy3.yaml.disabled} (100%) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml.disabled similarity index 100% rename from .github/workflows/pypy3.yaml rename to .github/workflows/pypy3.yaml.disabled From 8bfd91c7b4649060b5704ac3f092d62ae667ca11 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 055/333] renamed ubuntu_14.04.yaml to ubuntu_14.04.yaml.disabled --- .../workflows/{ubuntu_14.04.yaml => ubuntu_14.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_14.04.yaml => ubuntu_14.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_14.04.yaml rename to .github/workflows/ubuntu_14.04.yaml.disabled From a4139b161723a350733d1ce22aab019e536a4c80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 056/333] renamed ubuntu_16.04.yaml to ubuntu_16.04.yaml.disabled --- .../workflows/{ubuntu_16.04.yaml => ubuntu_16.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_16.04.yaml => ubuntu_16.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_16.04.yaml rename to .github/workflows/ubuntu_16.04.yaml.disabled From 46ab2bf3cf7fbb872f451c9d0590a9b2dc8ce803 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 057/333] renamed ubuntu_18.04.yaml to ubuntu_18.04.yaml.disabled --- .../workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml rename to .github/workflows/ubuntu_18.04.yaml.disabled From d3e2d0be996674fe5ccedc47a9bce75b436532fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 058/333] renamed ubuntu_20.04.yaml to ubuntu_20.04.yaml.disabled --- .../workflows/{ubuntu_20.04.yaml => ubuntu_20.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_20.04.yaml => ubuntu_20.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_20.04.yaml rename to .github/workflows/ubuntu_20.04.yaml.disabled From 1f744ad1dbf1d66fddae98a6244c7f1f35ece41b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 059/333] renamed ubuntu_github.yaml to ubuntu_github.yaml.disabled --- .../workflows/{ubuntu_github.yaml => ubuntu_github.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_github.yaml => ubuntu_github.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_github.yaml rename to .github/workflows/ubuntu_github.yaml.disabled From 72c62d30af56d108aa1554fb4884dd6abe635ba0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:02:48 +0100 Subject: [PATCH 060/333] renamed ubuntu_18.04.yaml.disabled to ubuntu_18.04.yaml --- .../workflows/{ubuntu_18.04.yaml.disabled => ubuntu_18.04.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml.disabled => ubuntu_18.04.yaml} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml.disabled rename to .github/workflows/ubuntu_18.04.yaml From 62d24176064c6b4dac09d71938d5f0d6e2b34c58 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:03:08 +0100 Subject: [PATCH 061/333] renamed ubuntu_20.04.yaml.disabled to ubuntu_20.04.yaml --- .../workflows/{ubuntu_20.04.yaml.disabled => ubuntu_20.04.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_20.04.yaml.disabled => ubuntu_20.04.yaml} (100%) diff --git a/.github/workflows/ubuntu_20.04.yaml.disabled b/.github/workflows/ubuntu_20.04.yaml similarity index 100% rename from .github/workflows/ubuntu_20.04.yaml.disabled rename to .github/workflows/ubuntu_20.04.yaml From 209f3bfd5641f436862ffca3dece34d0bf81b3fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:05:48 +0100 Subject: [PATCH 062/333] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5b9392d62..32ee3ce53 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,6 @@ [![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) -[![Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+14.04%22) -[![Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+16.04%22) [![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) From e550318685f87e7e2c25b400e634175915a8debe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:06:12 +0100 Subject: [PATCH 063/333] renamed ubuntu_github.yaml.disabled to ubuntu_github.yaml --- .../workflows/{ubuntu_github.yaml.disabled => ubuntu_github.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_github.yaml.disabled => ubuntu_github.yaml} (100%) diff --git a/.github/workflows/ubuntu_github.yaml.disabled b/.github/workflows/ubuntu_github.yaml similarity index 100% rename from .github/workflows/ubuntu_github.yaml.disabled rename to .github/workflows/ubuntu_github.yaml From c0e1bb251c0c9028b9f5b0907788e7766f346acf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:07:38 +0100 Subject: [PATCH 064/333] renamed ubuntu_18.04.yaml to ubuntu_18.04.yaml.disabled --- .../workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml rename to .github/workflows/ubuntu_18.04.yaml.disabled From cbf21a97aabc95628d71f773f08229cab9ac8614 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:10:29 +0100 Subject: [PATCH 065/333] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32ee3ce53..138775975 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,8 @@ [![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) -[![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) +[![Ubuntu 22.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2022.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+22.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) From a5ad66e4090134ecc51d4dbb869251e63f8f27c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:40 +0100 Subject: [PATCH 066/333] renamed centos.yaml to centos.yaml.disabled --- .github/workflows/{centos.yaml => centos.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos.yaml => centos.yaml.disabled} (100%) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml.disabled similarity index 100% rename from .github/workflows/centos.yaml rename to .github/workflows/centos.yaml.disabled From af02db5511f56a40b40a90702619c5dfd8f600a3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:40 +0100 Subject: [PATCH 067/333] renamed centos7.yaml to centos7.yaml.disabled --- .github/workflows/{centos7.yaml => centos7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos7.yaml => centos7.yaml.disabled} (100%) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml.disabled similarity index 100% rename from .github/workflows/centos7.yaml rename to .github/workflows/centos7.yaml.disabled From 5b91ee48587426db040ac1afb31fbfef8f1fbeff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:41 +0100 Subject: [PATCH 068/333] renamed centos8.yaml to centos8.yaml.disabled --- .github/workflows/{centos8.yaml => centos8.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos8.yaml => centos8.yaml.disabled} (100%) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml.disabled similarity index 100% rename from .github/workflows/centos8.yaml rename to .github/workflows/centos8.yaml.disabled From 79fa99d693ea366638e1ee6c38bd5dec65e4ee43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:28:28 +0100 Subject: [PATCH 069/333] updated README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 138775975..72b1c3aab 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,6 @@ [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) [![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) -[![CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22) -[![CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22) -[![CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22) [![Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Fedora%22) [![Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine%22) [![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) From 78ba04ce9d93aedfee00e09cb99c083884a53e1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:48:44 +0100 Subject: [PATCH 070/333] updated serf_event_handler.py --- serf_event_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serf_event_handler.py b/serf_event_handler.py index 455d7bee1..c0d5cacbb 100755 --- a/serf_event_handler.py +++ b/serf_event_handler.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.2' +__version__ = '0.2.3' class SerfEventHandler(CLI): @@ -95,7 +95,7 @@ def add_options(self): def enable_commands(self): if self.event in ['query', 'event']: - cmd = None + cmd = '' if self.event == 'query': cmd = self.query_name elif self.event == 'user': From f66248cc4545fe236327911857290247eba1a701 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:49:41 +0100 Subject: [PATCH 071/333] updated spark_json_to_avro.py --- spark_json_to_avro.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index 2922c7f7f..d53a4bffa 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -109,7 +109,6 @@ def run(self): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): df = sqlContext.read.json(json_file) else: From 8446d976d4efcee77aad458c5086d16c86a74255 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:53:59 +0100 Subject: [PATCH 072/333] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 72b1c3aab..e12942015 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools) [![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) From d80f92c71f9aefed3a161dc9cf719766ad4c1eef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:05:15 +0100 Subject: [PATCH 073/333] renamed docker_pytools_centos.yaml to *_centos.yaml.disabled --- .../{docker_pytools_centos.yaml => *_centos.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{docker_pytools_centos.yaml => *_centos.yaml.disabled} (100%) diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/*_centos.yaml.disabled similarity index 100% rename from .github/workflows/docker_pytools_centos.yaml rename to .github/workflows/*_centos.yaml.disabled From 930a2c341a40292160d34cfc827a6b8358f306a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:06:10 +0100 Subject: [PATCH 074/333] updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index e12942015..c301b354d 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,6 @@ [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) -[![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) [![Docker Build (Fedora)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml) [![Docker Build (Ubuntu)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml) From 2f629017ce6a08b0db62bff40c1a90f5f801ab53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:11:11 +0100 Subject: [PATCH 075/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 5e7dfbdc8..bccee5cd7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 5e7dfbdc8cea422b6856153eea3c01df4c84a60c +Subproject commit bccee5cd73f9bb86ab84609681defcea0ec23b9d From a109075a96e1d21f3dbe6cc21dd76b4a0706359e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:11:11 +0100 Subject: [PATCH 076/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4515bb958..47048ee0c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4515bb958f4b27c7c5854ba77ee665b41852dd5b +Subproject commit 47048ee0c39d1e88a10d09fec64333b2de8e906b From 5bdc0c0db940cbc74019b7b37508c6943e0b65f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:33:03 +0100 Subject: [PATCH 077/333] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index c301b354d..edb49e6b1 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,6 @@ [![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) [![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) -[![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) -[![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) [git.io/pytools](https://git.io/pytools) From e2f37bf682f28263a993209c5e52bcef272025d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:36:37 +0100 Subject: [PATCH 078/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bccee5cd7..995ad2de8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bccee5cd73f9bb86ab84609681defcea0ec23b9d +Subproject commit 995ad2de8486a96487e68c859c4f353a381b91d7 From 8bb7bc784d8f0ad98dbefd691ee8bc460f727330 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:36:37 +0100 Subject: [PATCH 079/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 47048ee0c..7c0ba3afe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 47048ee0c39d1e88a10d09fec64333b2de8e906b +Subproject commit 7c0ba3afe47f47af186cadab75b508a69d4411fe From 383114f4538c6ccc9f2443d5c53d96e6f2c3d526 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:39:36 +0100 Subject: [PATCH 080/333] added ubuntu_22.04.yaml --- .github/workflows/ubuntu_22.04.yaml | 85 +++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/ubuntu_22.04.yaml diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml new file mode 100644 index 000000000..b9bf17a95 --- /dev/null +++ b/.github/workflows/ubuntu_22.04.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 22.04 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_22.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:22.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} From 2769fba289020e9b94311b1f1bee3bcefc02f60e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:43:07 +0100 Subject: [PATCH 081/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 995ad2de8..fe25b15b0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 995ad2de8486a96487e68c859c4f353a381b91d7 +Subproject commit fe25b15b096e5e667b0451ebc9f68cc86c87384f From 6bfe3eb1a41357c469d3895b6739dfea28d7fab3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:43:07 +0100 Subject: [PATCH 082/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7c0ba3afe..0a225da5c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7c0ba3afe47f47af186cadab75b508a69d4411fe +Subproject commit 0a225da5c8aa09653d53dd1aaad545d200ec7e7a From 55827d1746db9186a49880e91d4f6d711058e7fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 18:50:55 +0100 Subject: [PATCH 083/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fe25b15b0..2295778d8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fe25b15b096e5e667b0451ebc9f68cc86c87384f +Subproject commit 2295778d8039333abc3e159754641b348cf79d94 From 5491f93ea4295dd3a87c7541a5182b146baadfe7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 18:50:56 +0100 Subject: [PATCH 084/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0a225da5c..6e5e84d30 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0a225da5c8aa09653d53dd1aaad545d200ec7e7a +Subproject commit 6e5e84d301329d0386b8a3468e4f7b34700d3e32 From a9c81105474dbfad815563507e21fa7b6f5d8785 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 19:07:39 +0100 Subject: [PATCH 085/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2295778d8..37dede02a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2295778d8039333abc3e159754641b348cf79d94 +Subproject commit 37dede02a5a184087798740bfbb36b90cf5ed685 From 3680e4c7ab65c6ef873bdf8151fe86efe68fd16a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 19:07:39 +0100 Subject: [PATCH 086/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6e5e84d30..5f2f181a9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6e5e84d301329d0386b8a3468e4f7b34700d3e32 +Subproject commit 5f2f181a96f884acd0340125aab776414c343b68 From 8865ba8731144538dd0baba23d1ccb787c5988a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 May 2023 17:50:27 +0100 Subject: [PATCH 087/333] added --unspace functionality --- center.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/center.py b/center.py index 4856ea0f7..b9cbac5f1 100755 --- a/center.py +++ b/center.py @@ -44,7 +44,8 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.2' +__version__ = '0.5.0' + class Center(CLI): @@ -56,7 +57,9 @@ def __init__(self): # this doesn't put enough spaces around ampersands, eg. in "Auth & Config" #self.re_bound = re.compile(r'(\b)') self.re_spaces = re.compile(r'(\s)') + self.re_multiple_spaces = re.compile(r'(\s){2}') self.re_chars = re.compile(r'([^\s])(?!\s)') + self.re_chars_spaced = re.compile(r'([^\s])\s') self.timeout_default = None def add_options(self): @@ -66,11 +69,16 @@ def add_options(self): help='No comment prefix handling') self.add_opt('-s', '--space', action='store_true', default=False, help='Space all chars out, makes bigger headings') + self.add_opt('-u', '--unspace', action='store_true', default=False, + help='Removes spaces betweeen chars out, the inverse of --space') def run(self): log_option('width', self.get_opt('width')) log_option('no comment prefix', self.get_opt('no_comment')) log_option('space chars', self.get_opt('space')) + log_option('unspace chars', self.get_opt('unspace')) + if self.get_opt('space') and self.get_opt('unspace'): + self.usage("--space and --unspace are mutually exclusive!") if self.args: self.process_line(' '.join(self.args)) else: @@ -83,6 +91,11 @@ def space(self, line): line = self.re_chars.sub(r'\1 ', line) return line + def unspace(self, line): + line = self.re_chars_spaced.sub(r'\1', line) + line = self.re_multiple_spaces.sub(r'\1', line) + return line + def process_line(self, line): char = '' if not line: @@ -102,9 +115,12 @@ def process_line(self, line): line = line.lstrip(char) if self.get_opt('space'): line = self.space(line) + if self.get_opt('unspace'): + line = self.unspace(line) line = line.strip() side = int(max((self.get_opt('width') - len(line)) / 2, 0)) print(char + ' ' * side + line) + if __name__ == '__main__': Center().main() From ad1c2e05e9ebe50dcc9829209328c2fa9c3b9016 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 May 2023 17:50:49 +0100 Subject: [PATCH 088/333] updated .pylintrc --- .pylintrc | 751 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 508 insertions(+), 243 deletions(-) diff --git a/.pylintrc b/.pylintrc index c20ac8f94..1c56d5de1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,386 +1,651 @@ -[MASTER] +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y L i n t C o n f i g +# ============================================================================ # + +# pylint --generate-rcfile >> .pylintrc + +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= -# Specify a configuration file. -#rcfile= +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 -# Pickle collected data for later comparisons. -persistent=yes +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 -# List of plugins (as comma separated values of python modules names) to load, +# List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= -# Use multiple processes to speed up Pylint. -jobs=1 +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. -optimize-ast=no +[BASIC] -[MESSAGES CONTROL] +# Naming style matching correct argument names. +argument-naming-style=snake_case -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= +# Naming style matching correct attribute names. +attr-naming-style=snake_case -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111,consider-using-f-string +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata -[REPORTS] +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text +# Naming style matching correct class attribute names. +class-attribute-naming-style=any -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= -# Tells whether to display a full report or only the messages -reports=yes +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= +# Naming style matching correct class names. +class-naming-style=PascalCase +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= -[BASIC] +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,input +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +# Naming style matching correct method names. +method-naming-style=snake_case -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ -# Regular expression matching correct function names -function-rgx=[A-Za-z_][A-Za-z0-9_]{2,30}$ +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Naming style matching correct variable names. +variable-naming-style=snake_case -# Regular expression matching correct constant names -const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ +[CLASSES] -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ +[DESIGN] -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of arguments for function / method. +max-args=5 -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of attributes for a class (see R0902). +max-attributes=7 -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of branch for function / method body. +max-branches=12 -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ +# Maximum number of locals for function / method body. +max-locals=15 -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 +# Maximum number of return / yield for function / method body. +max-returns=6 -[ELIF] +# Maximum number of statements in function / method body. +max-statements=50 -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] -# Maximum number of characters on a single line. -max-line-length=120 +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator -# Maximum number of lines in a module -max-module-lines=1000 +[IMPORTS] -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= [LOGGING] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + # Logging modules to check that the string format arguments are in logging -# function parameter format +# function parameter format. logging-modules=logging +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO +notes=FIXME, + XXX, + TODO +# Regular expression of note tags to take in consideration. +notes-rgx= -[SIMILARITIES] -# Minimum lines number of a similarity. -min-similarity-lines=4 +[REFACTORING] -# Ignore comments when computing similarities. -ignore-comments=yes +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 -# Ignore docstrings when computing similarities. -ignore-docstrings=yes +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error -# Ignore imports when computing similarities. -ignore-imports=no +[REPORTS] -[SPELLING] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= -# List of comma separated words that should not be checked. -spelling-ignore-words= +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= +# Tells whether to display a full report or only the messages. +reports=no -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no +# Activate the evaluation score. +score=yes -[TYPECHECK] +[SIMILARITIES] -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes +# Comments are removed from the similarity computation +ignore-comments=yes -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= +# Docstrings are removed from the similarity computation +ignore-docstrings=yes -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). This supports can work -# with qualified names. -# -# avoids the following error: -# -# pylint -E ./check_zookeeper_version.py -# -# E: 69,12: Instance of '_socketobject' has no 'sendall' member (no-member) -# E: 70,19: Instance of '_socketobject' has no 'recv' member (no-member) -# -ignored-classes=SQLObject,_socketobject +# Imports are removed from the similarity computation +ignore-imports=yes -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= +# Signatures are removed from the similarity computation +ignore-signatures=yes +# Minimum lines number of a similarity. +min-similarity-lines=4 -[VARIABLES] -# Tells whether we should check for unused import in __init__ files. -init-import=no +[SPELLING] -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: +# List of comma separated words that should not be checked. +spelling-ignore-words= -[CLASSES] +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs +[STRING] -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no -[DESIGN] -# Maximum number of arguments for function / method -max-args=5 +[TYPECHECK] -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager -# Maximum number of locals for function / method body -max-locals=15 +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= -# Maximum number of return / yield for function / method body -max-returns=6 +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes -# Maximum number of branch for function / method body -max-branches=12 +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes -# Maximum number of statements in function / method body -max-statements=50 +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init -# Maximum number of parents for a class (see R0901). -max-parents=7 +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace -# Maximum number of attributes for a class (see R0902). -max-attributes=7 +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin +# List of decorators that change the signature of a decorated function. +signature-mutators= -[IMPORTS] -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec +[VARIABLES] -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= +# List of names allowed to shadow builtins +allowed-redefined-builtins= +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb -[EXCEPTIONS] +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io From 64b326acbcea84535f033b706e6bfb5b289a5f32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 03:29:52 +0100 Subject: [PATCH 089/333] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index edb49e6b1..f064e78b0 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ [![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) -[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 0487abce888a1fdd84a18f853ef1871e17772147 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 090/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 37dede02a..09a50e7e1 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 37dede02a5a184087798740bfbb36b90cf5ed685 +Subproject commit 09a50e7e1810c856eff735279a83c00116a7fcd5 From 493c6d1afb562ca7eca054bde3d48fd90b9586c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 091/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5f2f181a9..c3ab2659f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5f2f181a96f884acd0340125aab776414c343b68 +Subproject commit c3ab2659f369a3af7ee16344ef52e222fcd7a555 From a0fed602600e01b0aa97e35e6824560e71de1bb9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 092/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 8d838daa6..0d8bd4872 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 8d838daa669dc05d497d5fb2fa1aef631c613acf +Subproject commit 0d8bd48723672bbfec6eb549c80e564b28725a7b From dd64db0434d294fc3bd53a2858e4da7920667009 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:04:10 +0100 Subject: [PATCH 093/333] added kics.config --- kics.config | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 kics.config diff --git a/kics.config b/kics.config new file mode 100644 index 000000000..5fafd16d2 --- /dev/null +++ b/kics.config @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-05 18:05:53 +0100 (Fri, 05 May 2023) +# +# vim:ts=2:sts=2:sw=2:et:filetype=yaml +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# K i c s C o n f i g +# ============================================================================ # + +# https://github.com/Checkmarx/kics/blob/master/docs/configuration-file.md + +--- +#path: assets/iac_samples +verbose: true +log-file: true +#type: +# - Dockerfile +# - Kubernetes +#queries-path: "assets/queries" +exclude-paths: + # ignore submodules - handle them in the source repos only + - bash-tools/ + - pylib/ + - sql/ + - templates/ +#output-path: "results" From fee21a0c50c83186949d1ae3253f0882ae642054 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 094/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 09a50e7e1..b2bcc4c93 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 09a50e7e1810c856eff735279a83c00116a7fcd5 +Subproject commit b2bcc4c93fcfc3e89421d8d264e6a863d623cf32 From 131da36d872ec1e3b2013de15589703b594f5323 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 095/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c3ab2659f..e137295ce 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c3ab2659f369a3af7ee16344ef52e222fcd7a555 +Subproject commit e137295ce7ae4c484c705a7e800c3577af823cef From 0ba9caa0aa08f5e044c5e340b195ad008944fb1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 096/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 0d8bd4872..73d5f3b1f 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 0d8bd48723672bbfec6eb549c80e564b28725a7b +Subproject commit 73d5f3b1fa54a3900493417637652f057fab3b61 From 5c929d22aa11b8553ec809a1b11172b933047ec2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 21:54:57 +0100 Subject: [PATCH 097/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b2bcc4c93..9112b3d95 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b2bcc4c93fcfc3e89421d8d264e6a863d623cf32 +Subproject commit 9112b3d957ce48300a139fc94c1da963c2fa8bf0 From 62046edaf17c3d87f6e7d1f281813e0ec1b2aaf2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 21:54:57 +0100 Subject: [PATCH 098/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e137295ce..6a6b9cf72 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e137295ce7ae4c484c705a7e800c3577af823cef +Subproject commit 6a6b9cf7211121fa2bb3d8f72dd1ef8ba19bcdf5 From 6f96ebf189863beae3e9612a6deb2743044dee24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 22:51:14 +0100 Subject: [PATCH 099/333] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f064e78b0..9c78001bb 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,8 @@ [![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) [![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) From afe9d7a35dc370d604111a9e15c824e4d63f6fe2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:48 +0100 Subject: [PATCH 100/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9112b3d95..a3750afbd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9112b3d957ce48300a139fc94c1da963c2fa8bf0 +Subproject commit a3750afbd00c67a0c497367be19bd807f05c242b From c9ac6b65bb38bbb6d5fa2375b5810d03fa9e46a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:48 +0100 Subject: [PATCH 101/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6a6b9cf72..5f2c84abb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6a6b9cf7211121fa2bb3d8f72dd1ef8ba19bcdf5 +Subproject commit 5f2c84abb142dfa218cc29fab89d11ddffa2011f From b2d609d7192f656094daf38192d4a396e5afee3a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:49 +0100 Subject: [PATCH 102/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 73d5f3b1f..590fd1816 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 73d5f3b1fa54a3900493417637652f057fab3b61 +Subproject commit 590fd18169dfed69dd5861d1b33d9c23b4a71a03 From 0341c77f80096ce8a3c7171d5e92ced882d973e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:25:24 +0100 Subject: [PATCH 103/333] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 9c78001bb..2ccdff67a 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,11 @@ [![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) From 472331bac11a4d00a39cb865dcc41540e3cdbc8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:29:06 +0100 Subject: [PATCH 104/333] updated .pylintrc --- .pylintrc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 1c56d5de1..b35a5fccc 100644 --- a/.pylintrc +++ b/.pylintrc @@ -448,7 +448,10 @@ disable=raw-checker-failed, suppressed-message, useless-suppression, deprecated-pragma, - use-symbolic-message-instead + use-symbolic-message-instead, + missing-function-docstring, + super-with-arguments, + consider-using-f-string # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 93b2c4f3353810ae7a53f4b6707ec9e1c371b7a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:30:27 +0100 Subject: [PATCH 105/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5f2c84abb..0f48c7572 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5f2c84abb142dfa218cc29fab89d11ddffa2011f +Subproject commit 0f48c7572c549bb109c3d946c7908048854dec73 From 2c42d10e551497e571f0297b36636f04bfb1025a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:45:22 +0100 Subject: [PATCH 106/333] updated .pylintrc --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index b35a5fccc..2006619da 100644 --- a/.pylintrc +++ b/.pylintrc @@ -449,6 +449,7 @@ disable=raw-checker-failed, useless-suppression, deprecated-pragma, use-symbolic-message-instead, + missing-class-docstring, missing-function-docstring, super-with-arguments, consider-using-f-string From 7a42f40f3dfa0d0a02658d82f630a59d3f5d5e8b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:47:54 +0100 Subject: [PATCH 107/333] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index fb56e18cf..93ea98595 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -166,7 +166,6 @@ def create_struct(arg): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') @@ -198,5 +197,6 @@ def create_struct(arg): # the databricks avro driver df.write.format('com.databricks.spark.avro').save(avro_dir) + if __name__ == '__main__': SparkCSVToAvro().main() From a282dda7d3eb16d5d8777db108bec344e3d90fff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:43 +0100 Subject: [PATCH 108/333] updated spark_avro_to_parquet.py --- spark_avro_to_parquet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py index 9300c4fd1..8528af1a6 100755 --- a/spark_avro_to_parquet.py +++ b/spark_avro_to_parquet.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 246d4df8036306782234ed6c1c4d47a7dee9d44e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:43 +0100 Subject: [PATCH 109/333] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index 93ea98595..d41717b4f 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -57,6 +57,7 @@ 'com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From dad1bce7ddc6cc5972ba5df65df8ff0138b1f659 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 110/333] updated spark_csv_to_parquet.py --- spark_csv_to_parquet.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py index 8e3b0da1c..e70f014f6 100755 --- a/spark_csv_to_parquet.py +++ b/spark_csv_to_parquet.py @@ -49,6 +49,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-csv_2.11:1.5.0 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error @@ -157,9 +158,7 @@ def create_struct(arg): if not isVersionLax(spark_version): die("Spark version couldn't be determined. " + support_msg('pytools')) - # pylint: disable=invalid-name - - df = None + # pylint: disable=invalid-name if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') From 49c66be7fd0cc50d196ba7e6c1f0a503c14c1771 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 111/333] updated spark_json_to_avro.py --- spark_json_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index d53a4bffa..db9d069ed 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 7fb4da436c7016412fe3b7426ab4ea06b676e2e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 112/333] updated spark_json_to_parquet.py --- spark_json_to_parquet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py index 2ed935917..6976ce153 100755 --- a/spark_json_to_parquet.py +++ b/spark_json_to_parquet.py @@ -43,6 +43,7 @@ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) sys.exit(4) pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 86727ce5e22f53fa4d84627865c4cad83dddf70e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 113/333] updated spark_parquet_to_avro.py --- spark_parquet_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py index 0186981f5..030816963 100755 --- a/spark_parquet_to_avro.py +++ b/spark_parquet_to_avro.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From ec22d9ca18fbd0ec75d1137a2ae11c8001247286 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:50 +0100 Subject: [PATCH 114/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a3750afbd..1b0b1149f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a3750afbd00c67a0c497367be19bd807f05c242b +Subproject commit 1b0b1149fabbbdb4a1c07cd25d2c1941034eb0a1 From f81d2e16df704a4fee44f20a3a48658b0e1f7c17 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:51 +0100 Subject: [PATCH 115/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0f48c7572..e53f962a7 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0f48c7572c549bb109c3d946c7908048854dec73 +Subproject commit e53f962a7d0eee7cf1f89ca98552d90c0d6226cb From 1643e01fddfd93d2c301a9a6ca925c1192389ef2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:51 +0100 Subject: [PATCH 116/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 590fd1816..3a74368ea 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 590fd18169dfed69dd5861d1b33d9c23b4a71a03 +Subproject commit 3a74368ead1544983ac21af7500b380b9fc20b9f From dbc5cec14dea46ef3f851d56988ab9173f8ace9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:01 +0100 Subject: [PATCH 117/333] updated Makefile --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index a67a168a9..13e4c9b27 100644 --- a/Makefile +++ b/Makefile @@ -86,12 +86,12 @@ init: # @$(MAKE) $@c %.pyc:: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing - @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ echo Generated $@ %.pyo:: %.py - @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile $< && \ echo && \ echo Generated $@ @@ -118,15 +118,15 @@ python: pylib @# only install pip packages not installed via system packages @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt @#$(SUDO_PIP) $(PIP) install -r requirements.txt - @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt + @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3"; fi - PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3" || : + @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3"; fi + PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3" || : @# for impyla @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || : @@ -154,7 +154,7 @@ python: pylib @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi @# Python >= 2.7 - won't build on 2.6, handle separately and accept failure - @PIP=$(PIP) bash-tools/python_pip_install.sh "ipython[notebook]" || : + @PIP=$(PIP) bash-tools/python/python_pip_install.sh "ipython[notebook]" || : @echo $(MAKE) pycompile @echo @@ -216,7 +216,7 @@ test: .PHONY: basic-test basic-test: test-lib - bash-tools/check_all.sh + bash-tools/checks/check_all.sh .PHONY: install install: build From bf4934fe8873bc1153ac2fef4f796a112fc93483 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:38 +0100 Subject: [PATCH 118/333] updated all.sh --- tests/all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/all.sh b/tests/all.sh index 33452eb7c..dbac7b8fb 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -31,7 +31,7 @@ section "Running PyTools ALL" cd "$srcdir/.."; # shellcheck disable=SC1090 # has to be included so that isExcluded function is inherited -. "$srcdir/../bash-tools/check_all.sh" +. "$srcdir/../bash-tools/checks/check_all.sh" #tests/test_yamllint.sh @@ -41,4 +41,4 @@ exit 0 # pyspark not found tests/help.sh -bash-tools/run_tests.sh +bash-tools/checks/run_tests.sh From 9a1f9be8ea238a1bc7cb04291f1eb743308bc09b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:47 +0100 Subject: [PATCH 119/333] updated anonymize_parallel.sh --- anonymize_parallel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 1f34f863a..abb7e9d21 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -68,7 +68,7 @@ for filename in $file_list; do echo "Removing any pre-existing parts:" rm -v "$filename".* 2>/dev/null || : echo - "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename" + "$srcdir/bash-tools/bin/split.sh" --parts "$parallelism" "$filename" echo "Anonymizing parts" for file_part in "$filename".*; do cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized" From 7251c68560d2f1f605475a8fb0df98e28c18353f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 11 May 2023 23:02:38 +0100 Subject: [PATCH 120/333] updated sonar-project.properties --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 270ec8ac7..7189b1a16 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -21,7 +21,7 @@ sonar.projectDescription=Python / Jython Tools sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues -sonar.links.ci=https://travis-ci.org/HariSekhon/devops-python-tools +sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions sonar.sources=. From 754947957d53ef5ed572bdcbb2b91a6f4f4b583c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 11 May 2023 23:59:10 +0100 Subject: [PATCH 121/333] updated README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2ccdff67a..56a02a41e 100644 --- a/README.md +++ b/README.md @@ -469,6 +469,10 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery +- [Jenkins](https://github.com/HariSekhon/Jenkins) - Advanced Jenkinsfile & Jenkins Groovy Shared Library + +- [GitHub-Actions](https://github.com/HariSekhon/GitHub-Actions) - GitHub Actions master template & GitHub Actions Shared Workflows library + - [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. - [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments From 01af37704aed0bcbe5eeb78d0bbcd3e849358b63 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 00:37:00 +0100 Subject: [PATCH 122/333] added trivy.yaml --- .github/workflows/trivy.yaml | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/trivy.yaml diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 000000000..5c9475308 --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# T r i v y +# ============================================================================ # + +--- +name: Trivy + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + trivy: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Trivy Filesystem Scan + uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From 0d560dd9f6634ad3ef668f2aa8ee52ab20d0f204 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 00:47:26 +0100 Subject: [PATCH 123/333] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 56a02a41e..aeab7f613 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) +[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) -[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) From 585e9050a905151a82746a4a8bfe42b993bf7ed6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 01:08:50 +0100 Subject: [PATCH 124/333] updated trivy.yaml --- .github/workflows/trivy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 5c9475308..8af2aeb39 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -1,6 +1,6 @@ # # Author: Hari Sekhon -# Date: Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) # # vim:ts=2:sts=2:sw=2:et # @@ -54,7 +54,7 @@ jobs: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' - name: Trivy Filesystem Scan + name: Trivy uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master with: debug: ${{ github.event.inputs.debug }} From 92b23f826de7fcdab498eb6b9e0ac26b306f09d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 01:41:06 +0100 Subject: [PATCH 125/333] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index aeab7f613..a41120672 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) [![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) From 9fe0d3ca28260ec2c4414a6f77c6ebbd0e5c390f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 07:15:53 +0100 Subject: [PATCH 126/333] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a41120672..36f322455 100644 --- a/README.md +++ b/README.md @@ -467,7 +467,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ## Related Repositories -- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 800+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 1000+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From b8c21da103ec580fb9b26ade93ad3eeb664cd2db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 14 May 2023 04:18:31 +0100 Subject: [PATCH 127/333] added grype.yaml --- .github/workflows/grype.yaml | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/grype.yaml diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml new file mode 100644 index 000000000..6a321f86e --- /dev/null +++ b/.github/workflows/grype.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G r y p e +# ============================================================================ # + +--- +name: Grype + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + Grype: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Grype + uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From a8c29b46dd6d8b0659fb64bbbc41c1d2cd6666e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 May 2023 02:06:58 +0100 Subject: [PATCH 128/333] updated center.py --- center.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/center.py b/center.py index b9cbac5f1..379353e68 100755 --- a/center.py +++ b/center.py @@ -44,7 +44,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class Center(CLI): @@ -104,7 +104,7 @@ def process_line(self, line): char = ' ' # preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it line = line.strip() - if isChars(line[0], '#'): + if line and isChars(line[0], '#'): char = line[0] line = line.lstrip(char) elif len(line) > 1 and isChars(line[0:1], '/'): From f9abc5f8321e73b34e0be59079823ce850b50362 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:24 +0100 Subject: [PATCH 129/333] removed codeship.yml --- codeship.yml | 58 ---------------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 codeship.yml diff --git a/codeship.yml b/codeship.yml deleted file mode 100644 index f466bd6a7..000000000 --- a/codeship.yml +++ /dev/null @@ -1,58 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2021-04-12 18:33:44 +0100 (Mon, 12 Apr 2021) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# ============================================================================ # -# C o d e S h i p -# ============================================================================ # - -# LEGACY - CloudBees has removed free plans as of 1st March 2023 - -# 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this - -# https://github.com/painless-software/codeship-yaml - -# Requires setting up the CodeShip commands like so: -# -# pip install codeship-yaml -# codeship-yaml -# -# or seaparately in sections: -# -# Project Settings > Test Settings > Setup Commands: -# -# pip install codeship-yaml -# codeship-yaml install -# -# Project Settings > Test Settings > Test Commands: -# -# codeship-yaml before_script script -# -# Project Settings > Deployment > (branch name) -# -# codeship-yaml after_success - ---- -install: - # these cause package installation breakages due to GPG or 403 errors, old addresses etc. - - sudo rm -fv -- /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list - - sudo rm -fv -- /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list - - sudo rm -fv -- /etc/apt/sources.list.d/www_apache_org_dist_cassandra_debian.list - - make -#before_script: -# - somecommand -script: - - make test -#after_success: -# - echo "Now we can deploy" From c4c513b66715f87bbed88320deda71205f84ce96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:32 +0100 Subject: [PATCH 130/333] removed wercker.yml --- wercker.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 wercker.yml diff --git a/wercker.yml b/wercker.yml deleted file mode 100644 index c27be30ed..000000000 --- a/wercker.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# https://devcenter.wercker.com/reference/wercker-yml/ - -box: debian - -build: - steps: - - script: - name: ci bootstrap - code: setup/ci_bootstrap.sh - - script: - name: init - code: make init - - script: - name: build - code: make ci - - script: - name: test - code: make test From 116cdfda1b85dbd8803e21a3e126c1d4d58d8ab6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:38 +0100 Subject: [PATCH 131/333] removed .werckerignore --- .werckerignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .werckerignore diff --git a/.werckerignore b/.werckerignore deleted file mode 100644 index ff8fb8247..000000000 --- a/.werckerignore +++ /dev/null @@ -1 +0,0 @@ -**/.md From 5ba24dc535c907d2d574d652da6517e3ec2cf75a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:08 +0100 Subject: [PATCH 132/333] moved .teamcity.vcs.json to teamcity/ --- .teamcity.vcs.json => teamcity/.teamcity.vcs.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.json => teamcity/.teamcity.vcs.json (100%) diff --git a/.teamcity.vcs.json b/teamcity/.teamcity.vcs.json similarity index 100% rename from .teamcity.vcs.json rename to teamcity/.teamcity.vcs.json From 1730b699555891aff16137749cfbd88f7559f21b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:09 +0100 Subject: [PATCH 133/333] moved .teamcity.vcs.oauth.json to teamcity/ --- .teamcity.vcs.oauth.json => teamcity/.teamcity.vcs.oauth.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.oauth.json => teamcity/.teamcity.vcs.oauth.json (100%) diff --git a/.teamcity.vcs.oauth.json b/teamcity/.teamcity.vcs.oauth.json similarity index 100% rename from .teamcity.vcs.oauth.json rename to teamcity/.teamcity.vcs.oauth.json From 21d86fe66caa9b86030bedd0155287ad263ba17f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:09 +0100 Subject: [PATCH 134/333] moved .teamcity.vcs.ssh.json to teamcity/ --- .teamcity.vcs.ssh.json => teamcity/.teamcity.vcs.ssh.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.ssh.json => teamcity/.teamcity.vcs.ssh.json (100%) diff --git a/.teamcity.vcs.ssh.json b/teamcity/.teamcity.vcs.ssh.json similarity index 100% rename from .teamcity.vcs.ssh.json rename to teamcity/.teamcity.vcs.ssh.json From 637c5ade2863627f66f26bd6ee354416c698e69d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 135/333] moved .concourse.yml to cicd/ --- .concourse.yml => cicd/.concourse.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .concourse.yml => cicd/.concourse.yml (100%) diff --git a/.concourse.yml b/cicd/.concourse.yml similarity index 100% rename from .concourse.yml rename to cicd/.concourse.yml From 424a095fedc3298d2ce34a63f6a4296b2acb8140 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 136/333] moved .gocd.yml to cicd/ --- .gocd.yml => cicd/.gocd.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .gocd.yml => cicd/.gocd.yml (100%) diff --git a/.gocd.yml b/cicd/.gocd.yml similarity index 100% rename from .gocd.yml rename to cicd/.gocd.yml From 2451e74fd432d4cdac907970369e2c9059608f3f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 137/333] moved buildspec.yml to cicd/ --- buildspec.yml => cicd/buildspec.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename buildspec.yml => cicd/buildspec.yml (100%) diff --git a/buildspec.yml b/cicd/buildspec.yml similarity index 100% rename from buildspec.yml rename to cicd/buildspec.yml From fca148ae783746b8452a26818344156c5a1bcdae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 138/333] moved cloudbuild.yaml to cicd/ --- cloudbuild.yaml => cicd/cloudbuild.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cloudbuild.yaml => cicd/cloudbuild.yaml (100%) diff --git a/cloudbuild.yaml b/cicd/cloudbuild.yaml similarity index 100% rename from cloudbuild.yaml rename to cicd/cloudbuild.yaml From caf90ee899ba2b558bdbab7d28c87ae28c5da7ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:41:46 +0100 Subject: [PATCH 139/333] removed shippable.yml --- shippable.yml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 shippable.yml diff --git a/shippable.yml b/shippable.yml deleted file mode 100644 index 1fa06bc06..000000000 --- a/shippable.yml +++ /dev/null @@ -1,50 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# http://docs.shippable.com/platform/workflow/config/ - -# http://docs.shippable.com/ci/advancedOptions/environmentVariables/ - -language: none - -branches: - only: - - master - -build: - ci: - # workaround to broken repos - # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB - # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 - # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed. - # DevOps-Python-tools/Makefile.in:272: recipe for target 'apt-packages' failed - - rm -fv -- /etc/apt/sources.list.d/cassandra.sources.list* - - rm -fv -- /etc/apt/sources.list.d/yarn.list* - # Basho repo is giving a '402 payment required' error - # https://github.com/Shippable/support/issues/5172 - - rm -fv -- /etc/apt/sources.list.d/basho_riak.list - #- shippable_retry make - - setup/ci_bootstrap.sh - - make init - - make ci - - make test - -integrations: - notifications: - - integrationName: email - type: email - on_success: never - on_failure: never - on_pull_request: never From 7d04cce80786cc6c4f278ce4a773f7d6a6f69ae3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:01 +0100 Subject: [PATCH 140/333] updated .appveyor.yml --- .appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index f4b36fba0..c459fd55a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A p p V e y o r C I +# ============================================================================ # + # https://www.appveyor.com/docs/appveyor-yml/ image: Ubuntu From 00b3cf9ae996e7b5a8f5bc7577adfd9b420b9381 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:02 +0100 Subject: [PATCH 141/333] updated pipeline.yml --- .buildkite/pipeline.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 93f047c02..eb0d641fd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u i l d K i t e C I +# ============================================================================ # + # BuildKite Pipeline # # add this command to the UI and it will read the rest of the steps from here: From 526798d5606da7d44e405fb9931b9d27c9c39b71 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:03 +0100 Subject: [PATCH 142/333] updated config.yml --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5dbde36b7..09026fc6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r c l e C I +# ============================================================================ # + # Master Template with more advanced config: # # https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml From 126ad67b87f9f2ada233929556a37d3ee479126e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:04 +0100 Subject: [PATCH 143/333] updated .cirrus.yml --- .cirrus.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index d544aeb85..6916e2577 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r r u s C I +# ============================================================================ # + # https://cirrus-ci.org/guide/writing-tasks/ container: From 243482c43161216eb1c278c64a0451f731347385 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:05 +0100 Subject: [PATCH 144/333] updated .drone.yml --- .drone.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.drone.yml b/.drone.yml index 4d36c9db2..d105ab57a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,6 +15,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# D r o n e C I +# ============================================================================ # + # https://docs.drone.io/quickstart/cli/ # # https://docs.drone.io/cli/install/ From 84d0b04dff32333bb7da8058de7c36b98bab6015 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:06 +0100 Subject: [PATCH 145/333] updated .gitlab-ci.yml --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 477b32c3e..625220b32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G i t L a b C I +# ============================================================================ # + # https://docs.gitlab.com/ee/ci/yaml/README.html +#include: '.gitlab/*.y*ml' + image: ubuntu:18.04 job: From c21088d44d170cf28533c2af5633c753b3a493ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:15:38 +0100 Subject: [PATCH 146/333] updated sonar-project.properties --- sonar-project.properties | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/sonar-project.properties b/sonar-project.properties index 7189b1a16..9b1539430 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon -# Date: 2016-07-19 18:18:27 +0100 (Tue, 19 Jul 2016) +# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016) # # https://github.com/HariSekhon/DevOps-Python-tools # @@ -12,21 +12,30 @@ # https://www.linkedin.com/in/HariSekhon # -sonar.projectName=DevOps Python Tools -sonar.projectKey=pytools +# ============================================================================ # +# S o n a r Q u b e +# ============================================================================ # + +sonar.host.url=https://sonarcloud.io + +# Required metadata +sonar.organization=harisekhon +sonar.projectName=DevOps-Python-tools +sonar.projectKey=HariSekhon_DevOps-Python-tools sonar.projectVersion=1.0 -sonar.projectDescription=Python / Jython Tools +sonar.projectDescription=DevOps-Python-tools sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions +# directories to scan (defaults to sonar-project.properties dir otherwise) sonar.sources=. #sonar.language=py sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/spark*/**/* +sonar.exclusions=**/tests/** From 2c93d263101b61483ad2189406aefe87efe53ecf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:39 +0100 Subject: [PATCH 147/333] updated sonar-project.properties --- sonar-project.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 9b1539430..319cc6897 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -38,4 +38,5 @@ sonar.sources=. sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/** +#sonar.exclusions=**/tests/** +sonar.exclusions=**/zookeeper-*/**/* From 3b6c0b0324ba01c6596adb589c86ad6790225fbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:50 +0100 Subject: [PATCH 148/333] updated semaphore.yml --- .semaphore/semaphore.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e86c4f36e..761ef0697 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S e m a p h o r e C I +# ============================================================================ # + # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ version: v1.0 From 7e975e2cda81ac8b0177401bc8766caf7db9bf6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:51 +0100 Subject: [PATCH 149/333] updated README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 36f322455..35a30a739 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://harisekhon.github.io/CI-CD/) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) -[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) -[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.concourse.yml) +[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) @@ -55,8 +55,8 @@ [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) -[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) -[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) +[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) +[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) [![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) From 97a95b048ddff13a76f31a2687f62a8c3f4ee10f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:52 +0100 Subject: [PATCH 150/333] updated azure-pipelines.yml --- azure-pipelines.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 76ad8f1f0..e26e7f760 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A z u r e D e v O p s P i p e l i n e +# ============================================================================ # + # https://aka.ms/yaml trigger: From e7b73f1e14052b5e152d8934fddd83e11929db02 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:53 +0100 Subject: [PATCH 151/333] updated buddy.yml --- buddy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/buddy.yml b/buddy.yml index f901a6425..e0b57451f 100644 --- a/buddy.yml +++ b/buddy.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u d d y C I +# ============================================================================ # + # https://buddy.works/docs/yaml/yaml-schema --- From 3f82815fe59aae0a425f870296787d9206c161c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:59 +0100 Subject: [PATCH 152/333] updated .concourse.yml --- cicd/.concourse.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cicd/.concourse.yml b/cicd/.concourse.yml index 42b806f26..92847121e 100644 --- a/cicd/.concourse.yml +++ b/cicd/.concourse.yml @@ -13,6 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o n c o u r s e C I +# ============================================================================ # + +# https://concourse-ci.org/golang-library-example.html + +# https://resource-types.concourse-ci.org/ +# https://concourse-ci.org/resource-types.html resources: - name: github icon: github-circle @@ -25,8 +33,7 @@ resources: # source: # interval: 1d -# https://concourse-ci.org/golang-library-example.html - +# https://concourse-ci.org/jobs.html jobs: - name: build public: false From 3851ded922052a5aa98677a3045fcbca03a13aa3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:00 +0100 Subject: [PATCH 153/333] updated .gocd.yml --- cicd/.gocd.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cicd/.gocd.yml b/cicd/.gocd.yml index f3ed9fd19..65583a818 100644 --- a/cicd/.gocd.yml +++ b/cicd/.gocd.yml @@ -12,6 +12,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G o C D +# ============================================================================ # + # https://github.com/tomzo/gocd-yaml-config-plugin#setup # https://docs.gocd.org/current/configuration/configuration_reference.html From f7dda886aad2cdcd0374877ca6b5ab9dae4bd2d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:00 +0100 Subject: [PATCH 154/333] updated codefresh.yml --- codefresh.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codefresh.yml b/codefresh.yml index 5326f6e51..7cddcaf7c 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e f r e s h C I +# ============================================================================ # + # https://codefresh.io/docs/docs/codefresh-yaml/ version: "1.0" From 0865b14f7edd4a67a348af0139713a706fa67e3a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:01 +0100 Subject: [PATCH 155/333] updated kics.config --- kics.config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kics.config b/kics.config index 5fafd16d2..51ac5712e 100644 --- a/kics.config +++ b/kics.config @@ -30,7 +30,15 @@ log-file: true exclude-paths: # ignore submodules - handle them in the source repos only - bash-tools/ + - github-actions/ + - haproxy-configs/ + - jenkins/ + - kubernetes-templates/ + - lib/ - pylib/ + - spotify-tools/ - sql/ + - sql-keywords/ - templates/ + - terraform-templates/ #output-path: "results" From 0ba05249f4a515b998110738105078ef6cb4d003 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:02 +0100 Subject: [PATCH 156/333] updated gocd_config_repo.json --- setup/gocd_config_repo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json index 4c1639b4d..ce63bf63d 100644 --- a/setup/gocd_config_repo.json +++ b/setup/gocd_config_repo.json @@ -12,7 +12,7 @@ "configuration": [ { "key": "file_pattern", - "value": "*.gocd.y*ml" + "value": "cicd/*.gocd.y*ml" } ], "rules": [ From f605276d04c77181fa4e8cf71c85cf6002752a35 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:05 +0100 Subject: [PATCH 157/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b0b1149f..36e0eabe2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b0b1149fabbbdb4a1c07cd25d2c1941034eb0a1 +Subproject commit 36e0eabe2b02207f51cd807fa6dafd18abdab234 From 37b79941c2329f2de041cab714d08851c1eaa5f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 158/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e53f962a7..af52e8795 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e53f962a7d0eee7cf1f89ca98552d90c0d6226cb +Subproject commit af52e87958755d74d9b96cd373aab8cc810ab6a6 From 0d7d8060f515735afca206920e275643934e4867 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 159/333] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index d4eedc0e2..b96c184fd 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit d4eedc0e2c68c9a43838c89a1d09abf10927a566 +Subproject commit b96c184fdda9b2ad607d222ec5663a087769b561 From d1ef74ffebc27e32ec9afeb5bca7debd4fedb64b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 160/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3a74368ea..d58140c42 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3a74368ead1544983ac21af7500b380b9fc20b9f +Subproject commit d58140c424a83a440fa1480e878c45a00920c0f4 From a3634ecf1e18a624af24814f4540cba15623a642 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:32 +0100 Subject: [PATCH 161/333] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 6b9eaf3b9..4dc3a09c3 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -11,8 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e O w n e r s +# ============================================================================ # + --- -name: Codeowners +name: CodeOwners on: push: From 79fdd3ef11e57c87b703255fc61789aedd083080 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:33 +0100 Subject: [PATCH 162/333] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 4d4ebd577..c7ff16fd4 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k S y n c +# ============================================================================ # + +# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch + --- name: Fork Sync From 6900c0b4ca31f052e94e7caffb77c16ec8dd7938 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:34 +0100 Subject: [PATCH 163/333] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 00479ecb7..94398083f 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -11,6 +11,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k U p d a t e P R +# ============================================================================ # + +# For a fork of the original repo, activate to keep its branches up to date via Pull Requests +# +# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date + --- name: Fork Update PR From 02e4b41f7117802742d8af57107da9aa09677982 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:35 +0100 Subject: [PATCH 164/333] updated json.yaml --- .github/workflows/json.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index 6ed4b92d1..f68e309e5 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# J S O N +# ============================================================================ # + +# Validate any JSON files found in the repo + --- name: JSON From 77f1608b618c75c87ad51006dfddba709aabdf37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:35 +0100 Subject: [PATCH 165/333] updated kics.yaml --- .github/workflows/kics.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index 399cc40f0..033f4331e 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -11,6 +11,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# K i c s +# ============================================================================ # + --- name: Kics From 70a101b1ae1095ba49d05a182d3a77a7eb352f73 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:36 +0100 Subject: [PATCH 166/333] updated shellcheck.yaml --- .github/workflows/shellcheck.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 98e53bfe4..753a6a495 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S h e l l C h e c k +# ============================================================================ # + +# Validate any shell scripts found in the repo + --- name: ShellCheck From 55a9f677eb8afdb95879e413b15da0b3d5543b27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:36 +0100 Subject: [PATCH 167/333] updated trivy.yaml --- .github/workflows/trivy.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 8af2aeb39..26f01fa94 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -15,6 +15,8 @@ # T r i v y # ============================================================================ # +# Scan files in the local repo + --- name: Trivy From b5f76c8c6c099ca81c0e0d6885d41f1195d85379 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:37 +0100 Subject: [PATCH 168/333] updated validate.yaml --- .github/workflows/validate.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 5d924b77b..94a8c007a 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# V a l i d a t i o n +# ============================================================================ # + +# Run all custom validations against files in the repo + --- name: Validation From 88ac6b6661928c5f272515fe0700576d04b6f279 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:37 +0100 Subject: [PATCH 169/333] updated xml.yaml --- .github/workflows/xml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml index f4d3b7035..8b07ed256 100644 --- a/.github/workflows/xml.yaml +++ b/.github/workflows/xml.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# X M L +# ============================================================================ # + +# Validate any XML files found in the repo + --- name: XML From 2c992b85d84bb9dd9ebd7ea2e3ab57dfafe13ab2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:38 +0100 Subject: [PATCH 170/333] updated yaml.yaml --- .github/workflows/yaml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index bee8be84c..d252f67d2 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# Y A M L +# ============================================================================ # + +# Validate any YAML files found in the repo + --- name: YAML From a2a575ec7dec99394aab6b8572f42953129e3e43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:49 +0100 Subject: [PATCH 171/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 36e0eabe2..815d75082 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 36e0eabe2b02207f51cd807fa6dafd18abdab234 +Subproject commit 815d75082d4faa8494e9500727ebd13e2c5abcae From 202d2ed1a86e2bd72f131fdd262eb6b88a846132 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 172/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index af52e8795..ae8aa8753 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit af52e87958755d74d9b96cd373aab8cc810ab6a6 +Subproject commit ae8aa875359ddea2c9b5cf6ccc422e12a0740949 From 25abbfdedc8b9832c507f970f7c1a8ef29643a27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 173/333] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index b96c184fd..e11486ebd 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit b96c184fdda9b2ad607d222ec5663a087769b561 +Subproject commit e11486ebd436802d22d1b88bda00c0a9eb3d41b6 From 24407a2473022336a734c95d262203780f75df87 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 174/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d58140c42..3af953c9a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d58140c424a83a440fa1480e878c45a00920c0f4 +Subproject commit 3af953c9ae196d4ba5e59637e9668bd1bc6e9953 From 82da083e55bda834adf1802587856c6ff5b8b95c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:01 +0100 Subject: [PATCH 175/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 815d75082..38901a555 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 815d75082d4faa8494e9500727ebd13e2c5abcae +Subproject commit 38901a555d2ce114f0e7df6d4a31a3fdc1a3dc41 From 4b432be8807c49caa2682b2f04ad492d403f2f0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:02 +0100 Subject: [PATCH 176/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ae8aa8753..aa20abd72 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ae8aa875359ddea2c9b5cf6ccc422e12a0740949 +Subproject commit aa20abd72ceef27723ad8c461f4d9321ad12e779 From e14414287476521c71efa1ca3cb1095d2886ca62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:02 +0100 Subject: [PATCH 177/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3af953c9a..15d1e3bc8 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3af953c9ae196d4ba5e59637e9668bd1bc6e9953 +Subproject commit 15d1e3bc8a594171b3c70ce0bf04979eceb8630a From baec6ac43c0a3bf03cd7d78bb35d9972d470c922 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 17:46:12 +0100 Subject: [PATCH 178/333] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35a30a739..eb8e5fe12 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Perl Lib](https://github.com/HariSekhon/lib) - Perl version of above library -- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, ArgoCD, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... +- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python & D2 source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, Jenkins, ArgoCD, Traefik, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... From b5db3cb80db2ddc5dcc980a844939766452dc308 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:32:47 +0400 Subject: [PATCH 212/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index da278a058..1438f07bd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit da278a05823c78709d831a4e6ba9cce4e5a481a7 +Subproject commit 1438f07bd9fe762752cfd986718e30797cff5650 From 8e50ccbc1c4ec668a1cea00152de7c700b584d24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:32:47 +0400 Subject: [PATCH 213/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a43798fe9..6f9e99389 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a43798fe986928bffe4fe5952efdce408d012562 +Subproject commit 6f9e993896910c320c571918a635dc049e757e73 From a2d58fac6ac19b0d1b5f663aab5939eefe2b9f28 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:33:56 +0400 Subject: [PATCH 214/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1438f07bd..b69edeada 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1438f07bd9fe762752cfd986718e30797cff5650 +Subproject commit b69edeadadf0774fc6444c552f3a89d407ca24d4 From 39a4591fa22b1a6c0ffe94e0f8451fa3a5a60f1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 May 2024 21:55:51 +0400 Subject: [PATCH 215/333] added .envrc --- .envrc | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..6fa553de2 --- /dev/null +++ b/.envrc @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# D i r e n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +# See more .envrc files in: +# +# https://github.com/HariSekhon/DevOps-Bash-tools +# +# .envrc-aws +# .envrc-gcp +# .envrc-kubernetes + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export COMPOSE_PROJECT_NAME="pytools" + +venv="$PWD/venv" + +if [ -f "$venv/bin/activate" ]; then + echo + echo "Local virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + echo +fi + +# read .env too +#dotenv From a0d2bc4abf677d283f992db3bab72461e888943f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:19:34 +0200 Subject: [PATCH 216/333] updated .envrc --- .envrc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.envrc b/.envrc index 6fa553de2..96d3b5c0f 100644 --- a/.envrc +++ b/.envrc @@ -35,18 +35,18 @@ set -euo pipefail export COMPOSE_PROJECT_NAME="pytools" -venv="$PWD/venv" - -if [ -f "$venv/bin/activate" ]; then - echo - echo "Local virtualenv directory found in: $venv" - echo - echo "Activating Virtualenv inside the directory: $venv" - - # shellcheck disable=SC1091 - source "$venv/bin/activate" - echo -fi +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + echo + fi +done # read .env too #dotenv From 0f9f99ff568ae21d903ceaaf0bf77b65c93632cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:20:41 +0200 Subject: [PATCH 217/333] updated .envrc --- .envrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.envrc b/.envrc index 96d3b5c0f..2aa9a4379 100644 --- a/.envrc +++ b/.envrc @@ -35,6 +35,7 @@ set -euo pipefail export COMPOSE_PROJECT_NAME="pytools" +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default for venv in "$PWD/venv" "$HOME/venv"; do if [ -f "$venv/bin/activate" ]; then echo From 92451e77812ddd6ace3c3ac401887fbeb2e1ad89 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:30 +0200 Subject: [PATCH 218/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b69edeada..4cf683ab7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b69edeadadf0774fc6444c552f3a89d407ca24d4 +Subproject commit 4cf683ab744f849c34d2c15ce255b18b368d6a11 From eef5f927cf6bc15c6d438706a37567529347dd7d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:31 +0200 Subject: [PATCH 219/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 99f5c866a..804fcd30f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 99f5c866aa5986c62d45de3b8da7043a28a63321 +Subproject commit 804fcd30fc9d05bf66fedae6001dbdef02dc8518 From 9616644dba9bf8d67673bcf02ae431af2395579d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:31 +0200 Subject: [PATCH 220/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6f9e99389..a43798fe9 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6f9e993896910c320c571918a635dc049e757e73 +Subproject commit a43798fe986928bffe4fe5952efdce408d012562 From 7dd10775f06d708e16930d91519297addaa33519 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 221/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4cf683ab7..8e8cf26f2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4cf683ab744f849c34d2c15ce255b18b368d6a11 +Subproject commit 8e8cf26f2c49b30371a35bbd844bae491fccfaf4 From aa3ef778e851dd980c962b4fafe2600dffd50123 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 222/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 804fcd30f..c88505e98 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 804fcd30fc9d05bf66fedae6001dbdef02dc8518 +Subproject commit c88505e98b4a59ffa961ac92a581be96233d3b9d From 749724f6c95f2baba13490258a14b79b45c79f2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 223/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a43798fe9..c2521be52 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a43798fe986928bffe4fe5952efdce408d012562 +Subproject commit c2521be525b4047222adb18263e5714e4171c483 From d8cbf436f9842d01a3bda55fdb095b2a4209b50d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:36:09 +0200 Subject: [PATCH 224/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c88505e98..545ce8b83 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c88505e98b4a59ffa961ac92a581be96233d3b9d +Subproject commit 545ce8b838428cb8e3ae8e883130a5fe251bd83d From 21f534993310e30479c38661de27c0ffc350946e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:38:47 +0200 Subject: [PATCH 225/333] updated .envrc --- .envrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.envrc b/.envrc index 2aa9a4379..9073c82fd 100644 --- a/.envrc +++ b/.envrc @@ -14,7 +14,7 @@ # # ============================================================================ # -# D i r e n v +# P y t h o n D i r E n v # ============================================================================ # # .envrc to auto-load the virtualenv inside the 'venv' directory if present From 82154119348fccba6d13f674c336bdc127c4ac44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:39:28 +0200 Subject: [PATCH 226/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8e8cf26f2..6d8f6ebc0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8e8cf26f2c49b30371a35bbd844bae491fccfaf4 +Subproject commit 6d8f6ebc0c93dad1306f35844b931c635cd1f9ce From 899a4cea79947f03e8e0862345914ddf47e68bee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:39:28 +0200 Subject: [PATCH 227/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 545ce8b83..76e9e28d1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 545ce8b838428cb8e3ae8e883130a5fe251bd83d +Subproject commit 76e9e28d1b3a016dc64d1ccf92c7a967a74c48aa From a63287671e5471c97794e9c19f41b676d3d12e81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 228/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6d8f6ebc0..9f1a8c533 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6d8f6ebc0c93dad1306f35844b931c635cd1f9ce +Subproject commit 9f1a8c53368a47e43e6f022f236e40635b78aa8f From 18cc287778458c4bfcaea0c12fca330302e6982c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 229/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 76e9e28d1..e2982d2f4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 76e9e28d1b3a016dc64d1ccf92c7a967a74c48aa +Subproject commit e2982d2f4c07d0252d59897d9002235f6c0a7f6c From 1f6f01941062e293ee1c8898e884bafca470dccc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 230/333] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 39d0eccb4..8597f692e 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 39d0eccb4c160fcee9f09df9bb53c911ca2a387f +Subproject commit 8597f692e4e4abd59b58fb24c7fa79449a75c8c0 From e805b2ab2933e6a05cfb0fbd0f6dfbabf5e50c3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 231/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c2521be52..5174a5073 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c2521be525b4047222adb18263e5714e4171c483 +Subproject commit 5174a5073dbe25c8e5b1ac603b7b4f61b6ac7623 From 86d645a4f49915bd3032fcc9fee866f0a437eb94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:40:47 +0200 Subject: [PATCH 232/333] updated .envrc --- .envrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.envrc b/.envrc index 9073c82fd..51de39abb 100644 --- a/.envrc +++ b/.envrc @@ -46,6 +46,7 @@ for venv in "$PWD/venv" "$HOME/venv"; do # shellcheck disable=SC1091 source "$venv/bin/activate" echo + break fi done From c6fcf3037124d84f6f0b9ce569993fe41eb76181 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:41:19 +0200 Subject: [PATCH 233/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9f1a8c533..f32bb01be 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9f1a8c53368a47e43e6f022f236e40635b78aa8f +Subproject commit f32bb01be9e59606bf4a292c0a3a7e2ef1ac4540 From 15816a21b449f0af739eb5985b61fe7ab9f9f475 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:41:19 +0200 Subject: [PATCH 234/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e2982d2f4..29fbc0f2d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e2982d2f4c07d0252d59897d9002235f6c0a7f6c +Subproject commit 29fbc0f2da0c8d839898eb90b2d3a26c72ec85c8 From 959c4ce33775ecf588e4ad5794d1c340752510a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:33:25 +0200 Subject: [PATCH 235/333] updated .flake8 --- .flake8 | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index e31fb949f..de51ee708 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,32 @@ +# +# Author: Hari Sekhon +# Date: Mon Oct 21 15:57:10 2019 +0100 +# +# vim:ts=4:sts=4:sw=4:et +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# F l a k e 8 C o n f i g +# ============================================================================ # + +# https://flake8.pycqa.org/en/latest/user/configuration.html + [flake8] -ignore = E265,E402,F401 + max-line-length = 120 + +ignore = E265, + E402, + F401 + exclude = test*/* + max-complexity = 10 From 06d5e9469a49cf33c0c618ac5e05fbd4343dd4c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 236/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f32bb01be..057b80266 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f32bb01be9e59606bf4a292c0a3a7e2ef1ac4540 +Subproject commit 057b80266f1bb0f3e36af2ae7f8fa228afc09074 From 49075c27e13a166fc5ac4dfd5582dfc9c936b928 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 237/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 29fbc0f2d..fdb270866 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 29fbc0f2da0c8d839898eb90b2d3a26c72ec85c8 +Subproject commit fdb2708661b5a58fdba4b7ba514246aa929e6c02 From cd4e61aa6c5d4ff4181342e5d8f8581a1b135493 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 238/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 5174a5073..d94994282 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 5174a5073dbe25c8e5b1ac603b7b4f61b6ac7623 +Subproject commit d94994282274abf4279444fd85c0674a382e6696 From ec92f71bad8c41778516345018fd6295861aa67f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:17:37 +0300 Subject: [PATCH 239/333] added .envrc-python --- .envrc-python | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .envrc-python diff --git a/.envrc-python b/.envrc-python new file mode 100644 index 000000000..6bc2d65b1 --- /dev/null +++ b/.envrc-python @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y t h o n D i r E n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + break + fi +done + +# read .env too +#dotenv From bae58f9cbadd09648d416c363f73e5ae1d5dd582 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:19:10 +0300 Subject: [PATCH 240/333] updated .envrc --- .envrc | 145 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 18 deletions(-) diff --git a/.envrc b/.envrc index 51de39abb..7e481411c 100644 --- a/.envrc +++ b/.envrc @@ -14,41 +14,150 @@ # # ============================================================================ # -# P y t h o n D i r E n v +# D i r E n v # ============================================================================ # -# .envrc to auto-load the virtualenv inside the 'venv' directory if present - # https://direnv.net/man/direnv-stdlib.1.html -# See more .envrc files in: -# -# https://github.com/HariSekhon/DevOps-Bash-tools +# See Also: # # .envrc-aws # .envrc-gcp # .envrc-kubernetes +# direnv stdlib - loads .envrc from parent dir up to / +# +# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. +# +# bypasses security authorization though - use with care +#source_up +# +# source_up must be loaded before set -u otherwise gets this error: +# +# direnv: loading .envrc +# /bin/bash: line 226: $1: unbound variable +# +# source_up causes this error is up .envrc is found in parent directories: +# +# direnv: No ancestor .envrc found + set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +src="$(readlink -f "${BASH_SOURCE[0]}")" +srcdir="$(cd "$(dirname "$src")" && pwd)" -export COMPOSE_PROJECT_NAME="pytools" +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # -# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default -for venv in "$PWD/venv" "$HOME/venv"; do - if [ -f "$venv/bin/activate" ]; then - echo - echo "Virtualenv directory found in: $venv" - echo - echo "Activating Virtualenv inside the directory: $venv" +# Automatically install Pre-Commit Git hooks if not already present - # shellcheck disable=SC1091 - source "$venv/bin/activate" +if [ -f .pre-commit-config.yaml ]; then + if [ -d .git ]; then + if ! [ -f .git/hooks/pre-commit ]; then + pre-commit install + fi + fi +fi + +# ============================================================================ # +# D o c k e r C o m p o s e +# ============================================================================ # + +export COMPOSE_PROJECT_NAME="DevOps-Python-tools" + +# ============================================================================ # +# G i t H u b +# ============================================================================ # + +#export GITHUB_ORGANIZATION=HariSekhon + +# ============================================================================ # +# A n s i b l e +# ============================================================================ # + +# use the local repo's ansible.cfg rather than: +# +# $PWD/ansible.cfg +# ~/.ansible.cfg +# /etc/ansible/ansible.cfg +# +# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used +#export ANSIBLE_CONFIG="/path/to/ansible.cfg" + +# ============================================================================ # +# C l o u d f l a r e +# ============================================================================ # + +#export CLOUDFLARE_EMAIL=hari@... +#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens +#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module + +# export the variables for terraform +#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" +#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above + +# ============================================================================ # +# P y t h o n , A W S , G C P , T e r r a f o r m +# ============================================================================ # + +# XXX: safer to bring all these external .envrc inline if you're worried about changes +# to it bypassing 'direnv allow' authorization +load_if_exists(){ + # first arg is a path to a .envrc + # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes + # to pass the context name 'docker-desktop' to switch to + local envrc="$1" + shift + if ! [[ "$envrc" =~ ^/ ]]; then + envrc="$srcdir/$envrc" + fi + if [ -f "$envrc" ]; then + # prevent looping on symlinks to this .envrc if given + if [ "$(readlink "$envrc")" = "$src" ]; then + return + fi echo - break + echo "Loading $envrc" + # shellcheck disable=SC1090,SC1091 + . "$envrc" "$@" fi +} + +#load_if_exists ~/.envrc + + #.envrc-aws \ + #.envrc-gcp \ + #.envrc-terraform \ +# shellcheck disable=SC2043 +for envrc in \ + .envrc-python \ + ; do + load_if_exists "$envrc" done +if [[ "$PWD" =~ /aws/ ]]; then + load_if_exists .envrc-aws +fi + +if [[ "$PWD" =~ /gcp/ ]]; then + load_if_exists .envrc-gcp +fi + +if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then + load_if_exists .envrc-terraform +fi + +# ============================================================================ # +# K u b e r n e t e s +# ============================================================================ # + +if [ -f "$srcdir/.envrc-kubernetes" ]; then + load_if_exists .envrc-kubernetes docker-desktop +fi + +# ============================================================================ # + +echo # read .env too #dotenv From 13b3cde4939345bbb49086f16b627056958b2329 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:51:24 +0300 Subject: [PATCH 241/333] added .pre-commit-config.yaml --- .pre-commit-config.yaml | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..f3c5b1608 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,72 @@ +# +# Author: Hari Sekhon +# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024) +# +# vim:ts=2:sts=2:sw=2:et +# +# https///github.com/HariSekhon/Templates +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +--- +fail_fast: false +#exclude: *.tmp$ + +repos: + + # will accept anything that 'git clone' understands + # this means you can set this to a local git repo to develop your own hook repos interactively + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + # Common errors + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: forbid-new-submodules + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + + - repo: https://github.com/psf/black + rev: 24.8.0 + hooks: + - id: black + + # Git secrets Leaks + - repo: https://github.com/awslabs/git-secrets.git + # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error: + # + # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file + # + rev: 5357e18 + hooks: + - id: git-secrets + + - repo: https://github.com/markdownlint/markdownlint + rev: v0.12.0 + hooks: + - id: markdownlint + name: Markdownlint + description: Run markdownlint on your Markdown files + entry: mdl + language: ruby + files: \.(md|mdown|markdown)$ From 5c145c7d2e6459c61a9a55c0f6f7047e009ccbda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:57:50 +0300 Subject: [PATCH 242/333] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3c5b1608..2b154867f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https///github.com/HariSekhon/Templates +# https///github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # From c8c6623a14f7482ab9b222f63074f0bcbb55e657 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 20:33:29 +0300 Subject: [PATCH 243/333] updated .envrc --- .envrc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.envrc b/.envrc index 7e481411c..da56f607a 100644 --- a/.envrc +++ b/.envrc @@ -52,11 +52,11 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present -if [ -f .pre-commit-config.yaml ]; then - if [ -d .git ]; then - if ! [ -f .git/hooks/pre-commit ]; then - pre-commit install - fi +if [ -f .pre-commit-config.yaml ] && + [ -d .git ] && + type -P pre-commit &>/dev/null; then + if ! [ -f .git/hooks/pre-commit ]; then + pre-commit install fi fi From 9a5c385d480ecf1e05aacf4e0e753af684f9e262 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 00:12:38 +0300 Subject: [PATCH 244/333] updated .editorconfig --- .editorconfig | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 73c19b202..462f53206 100644 --- a/.editorconfig +++ b/.editorconfig @@ -40,7 +40,14 @@ end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true -[*.md] +[{*.md,*.hcl,*.tf,*.tfvars}] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.yml,*.yaml] indent_size = 2 indent_style = space end_of_line = lf From f410868fe774c96117e790f2bb7f6f2bd3fd9af6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 01:49:04 +0300 Subject: [PATCH 245/333] updated .envrc --- .envrc | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index da56f607a..920b4e16d 100644 --- a/.envrc +++ b/.envrc @@ -98,7 +98,7 @@ export COMPOSE_PROJECT_NAME="DevOps-Python-tools" #export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above # ============================================================================ # -# P y t h o n , A W S , G C P , T e r r a f o r m +# Load External Envrc Files If Present # ============================================================================ # # XXX: safer to bring all these external .envrc inline if you're worried about changes @@ -124,8 +124,15 @@ load_if_exists(){ fi } +# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc +# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses +# to install to system Python) #load_if_exists ~/.envrc +# ============================================================================ # +# P y t h o n +# ============================================================================ # + #.envrc-aws \ #.envrc-gcp \ #.envrc-terraform \ @@ -136,14 +143,26 @@ for envrc in \ load_if_exists "$envrc" done +# ============================================================================ # +# A W S +# ============================================================================ # + if [[ "$PWD" =~ /aws/ ]]; then load_if_exists .envrc-aws fi +# ============================================================================ # +# G C P +# ============================================================================ # + if [[ "$PWD" =~ /gcp/ ]]; then load_if_exists .envrc-gcp fi +# ============================================================================ # +# T e r r a f o r m +# ============================================================================ # + if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then load_if_exists .envrc-terraform fi From 283a9a3ca15ea2293c9ae2ed08a9ac77aa7d0bfa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 12:06:01 +0300 Subject: [PATCH 246/333] updated .envrc --- .envrc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.envrc b/.envrc index 920b4e16d..15bc469ee 100644 --- a/.envrc +++ b/.envrc @@ -52,10 +52,23 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present +if ! type -P pre-commit &>/dev/null && + uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now..." + echo + brew install pre-commit + echo +fi + if [ -f .pre-commit-config.yaml ] && - [ -d .git ] && - type -P pre-commit &>/dev/null; then - if ! [ -f .git/hooks/pre-commit ]; then + type -P pre-commit &>/dev/null && + git rev-parse --is-inside-work-tree &>/dev/null; then + if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + echo + echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." + echo pre-commit install fi fi From f09fdd3279da8103197fc4be455be86eea23e2df Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 14 Aug 2024 23:17:32 +0200 Subject: [PATCH 247/333] updated .envrc --- .envrc | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.envrc b/.envrc index 15bc469ee..7555ecba9 100644 --- a/.envrc +++ b/.envrc @@ -52,14 +52,20 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present -if ! type -P pre-commit &>/dev/null && - uname -s | grep -q Darwin && - type -P brew &>/dev/null; then - echo - echo "Pre-commit is not installed - installing now..." - echo - brew install pre-commit - echo +if ! type -P pre-commit &>/dev/null; then + if uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Homebrew..." + echo + brew install pre-commit + echo + elif type -P pip &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Pip..." + echo + pip install pre-commit + fi fi if [ -f .pre-commit-config.yaml ] && @@ -188,6 +194,8 @@ if [ -f "$srcdir/.envrc-kubernetes" ]; then load_if_exists .envrc-kubernetes docker-desktop fi +# ============================================================================ # +# . E n v # ============================================================================ # echo From d31b9e28d672eedc9624cfc2890daf982fd6acd9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 15 Aug 2024 16:35:10 +0200 Subject: [PATCH 248/333] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 503347f5d..cc918aa8d 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,11 @@ [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) [![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) -[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) + [![ShellCheck](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml) [![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) From f3c5690d32c077c52b108ca1f83f88fbae06cfbe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 22 Aug 2024 02:22:09 +0200 Subject: [PATCH 249/333] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b154867f..df14fb7d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,5 +68,6 @@ repos: name: Markdownlint description: Run markdownlint on your Markdown files entry: mdl + args: [-s, .markdownlint.rb] language: ruby files: \.(md|mdown|markdown)$ From 4546c7688be6a4e0a12b333b8fa123c0aae7a91c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 22 Aug 2024 02:22:10 +0200 Subject: [PATCH 250/333] added .markdownlint.rb --- .markdownlint.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .markdownlint.rb diff --git a/.markdownlint.rb b/.markdownlint.rb new file mode 100644 index 000000000..e69de29bb From db58f5309bcf6fe5d145bb65afdc501b0c1494d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:18 +0200 Subject: [PATCH 251/333] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df14fb7d9..9e9c1c291 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,6 +68,6 @@ repos: name: Markdownlint description: Run markdownlint on your Markdown files entry: mdl - args: [-s, .markdownlint.rb] + args: [-s, .mdl.rb] language: ruby files: \.(md|mdown|markdown)$ From 0839b2b170aa75b94c39263577660b3cac5e25ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:19 +0200 Subject: [PATCH 252/333] added .mdlrc --- .mdlrc | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .mdlrc diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 000000000..27e5b6895 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,5 @@ +mdlrc_dir = File.expand_path('..', __FILE__) + +style_file = File.join(mdlrc_dir, '.mdl.rb') + +style style_file From c8ec774ef167b959685256791528eba64dbd8b32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:19 +0200 Subject: [PATCH 253/333] added .mdl.rb --- .mdl.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .mdl.rb diff --git a/.mdl.rb b/.mdl.rb new file mode 100644 index 000000000..db6a3373f --- /dev/null +++ b/.mdl.rb @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# vim:ts=4:sts=4:sw=4:et:filetype=ruby +# +# Author: Hari Sekhon +# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +all +#exclude_rule 'MD001' +#exclude_rule 'MD003' +#exclude_rule 'MD005' +exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af +#exclude_rule 'MD012' +exclude_rule 'MD013' # long lines cannot be split if they are URLs +#exclude_rule 'MD022' +#exclude_rule 'MD025' +#exclude_rule 'MD031' +#exclude_rule 'MD032' +exclude_rule 'MD033' # inline HTML is important for formatting +#exclude_rule 'MD039' +#exclude_rule 'MD056' From cfa61d4d041dba2306dfdfbf568226209d656f56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:36:37 +0200 Subject: [PATCH 254/333] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index ab20cfa8d..0b52378ff 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -52,7 +52,7 @@ retry(){ if [ "$(uname -s)" = Darwin ]; then echo "Bootstrapping Mac" # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make) - #retry "$srcdir/install_homebrew.sh" + #retry "$srcdir/../install/install_homebrew.sh" if command -v brew 2>&1; then # fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do From 63bde7fef5cf084fa7fe4a0a212d60d275f46ef2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 255/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 057b80266..e7c73c3c4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 057b80266f1bb0f3e36af2ae7f8fa228afc09074 +Subproject commit e7c73c3c436c41901d1b51c872e5c3e2b22e2b07 From 16a294f584f532cde95b2df7401cc50234bdd3d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 256/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fdb270866..092819070 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fdb2708661b5a58fdba4b7ba514246aa929e6c02 +Subproject commit 092819070eda268f0a4c6f8b373290c09a9de421 From c91a96c1ebd6a698d568d6a36d758872da9804b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 257/333] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 8597f692e..ace6d6400 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 8597f692e4e4abd59b58fb24c7fa79449a75c8c0 +Subproject commit ace6d6400c1f647f2e96bd6dc9b730cf1232847a From b53766eb78df127afdcc81e3dde0a50a8f9dabe8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:20 +0200 Subject: [PATCH 258/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d94994282..b8d565217 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d94994282274abf4279444fd85c0674a382e6696 +Subproject commit b8d565217a111514a2e061cec896f258712985fb From 82e9390f13636322f39891aa5cc372a4496428c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 17:15:58 +0200 Subject: [PATCH 259/333] updated README --- README.md | 196 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 116 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index cc918aa8d..4fd1177dc 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) -[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/HariSekhon/) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) -- [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... +### Knowledge -- [The Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) -- [Nagios Plugin Kafka](https://github.com/HariSekhon/Nagios-Plugin-Kafka) - Kafka API pub/sub Nagios Plugin written in Scala with Kerberos support + -- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python & D2 source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, Jenkins, ArgoCD, Traefik, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... +### DevOps Code -- [Knowledge-Base](https://github.com/HariSekhon/Knowledge-Base) - IT Knowledge Base from 20 years in DevOps, Linux, Cloud, Big Data, AWS, GCP etc. +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Python-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Perl-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Perl-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Golang-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Golang-tools) -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +### Containerization -- https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) -### Stargazers over time +### CI/CD -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) -[git.io/python-tools](https://git.io/python-tools) +### DBA - SQL -[git.io/pytools](https://git.io/pytools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) + +### DevOps Reloaded + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) + +### Misc + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) + +The rest of my original source repos are +[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). + +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). + + +![](https://hit.yhype.me/github/profile?user_id=2211051) + + From e27764e4a9ed13170d3585259ea6267be3b7f62d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 17:17:33 +0200 Subject: [PATCH 260/333] updated README --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4fd1177dc..5fa8938af 100644 --- a/README.md +++ b/README.md @@ -534,14 +534,17 @@ Does nothing: [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) -### Misc +### Templates +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) + +### Misc + [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) From 6259d5a9e744abde66b0ff91c15b194e8104b075 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:19:58 +0200 Subject: [PATCH 261/333] updated .mdl.rb --- .mdl.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/.mdl.rb b/.mdl.rb index db6a3373f..f8f9b004c 100644 --- a/.mdl.rb +++ b/.mdl.rb @@ -25,5 +25,6 @@ #exclude_rule 'MD031' #exclude_rule 'MD032' exclude_rule 'MD033' # inline HTML is important for formatting +exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines #exclude_rule 'MD039' #exclude_rule 'MD056' From e4eac61b95c830eb6ee80027214d72e9be9dcd6e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:30:55 +0200 Subject: [PATCH 262/333] updated README --- README.md | 176 +++++++++++++++++++++++++++--------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 5fa8938af..955eb8cb6 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Cloud & Big Data Contractor, United Kingdom [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/)
*(you're welcome to connect with me on LinkedIn)* -**Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries** +**Make sure you run `make update` if updating and not just `git pull` as you will often need the latest library submodule and possibly new upstream libraries** ## Quick Start @@ -163,14 +163,14 @@ Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://git ### Usage -All programs come with a ```--help``` switch which includes a program description and the list of command line options. +All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. ### DevOps Python Tools - Inventory - Linux: - - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -179,115 +179,115 @@ Environment variables are supported for convenience and also to hide credentials - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` - - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - ```find_active_server.py``` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` + - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [Amazon Web Services](https://aws.amazon.com/): - - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all docker images eg. `docker_pull_all_images_all_tags.sh` + - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` - [Spark](https://spark.apache.org/) & Data Format Converters: - - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter - - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas - - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```xml_to_json.py``` - XML to JSON converter - - ```json_to_xml.py``` - JSON to XML converter - - ```json_to_yaml.py``` - JSON to YAML converter - - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - - ```yaml_to_json.py``` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) - - see also ```validate_*.py``` further down for all these formats and more + - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter + - `spark_parquet_to_avro.py` - PySpark Parquet => Avro converter + - `spark_csv_to_avro.py` - PySpark CSV => Avro converter, supports both inferred and explicit schemas + - `spark_csv_to_parquet.py` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas + - `spark_json_to_avro.py` - PySpark JSON => Avro converter + - `spark_json_to_parquet.py` - PySpark JSON => Parquet converter + - `xml_to_json.py` - XML to JSON converter + - `json_to_xml.py` - JSON to XML converter + - `json_to_yaml.py` - JSON to YAML converter + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) + - see also `validate_*.py` further down for all these formats and more - [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: - [Ambari](https://hortonworks.com/apache/ambari/): - - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API + - `ambari_blueprints.py` - Blueprint cluster templating and deployment tool using Ambari API - list blueprints - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool - - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API - - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable using this tool + - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts + - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API + - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - ```hdfs_find_replication_factor_1.py``` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - ```hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - ```hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output The following programs can all optionally filter by database / table name regex: - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables - - ```hive_tables_row_column_counts.py``` / ```impala_tables_row_column_counts.py``` - outputs tables row and column counts. Useful for finding unusually big tables - - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_metadata.py``` / ```impala_tables_metadata.py``` - outputs for each table the matching regex metadata DDL property from describe table - - ```hive_tables_locations.py``` / ```impala_tables_locations.py``` - outputs for each table its data location + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL property from describe table + - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables - - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop -- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig_udfs.jy` - Pig Jython UDFs for Hadoop +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required - - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node + - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA + - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA + - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node + - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator + - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server + - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server + - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node + - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- ```selenium_hub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - - ```validate_*.py``` - validate files, directory trees and/or standard input streams + - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: - Avro - CSV @@ -326,7 +326,7 @@ Download the DevOps Python Tools and Pylib git repos as zip files: -Unzip both and move Pylib to the ```pylib``` folder under DevOps Python Tools. +Unzip both and move Pylib to the `pylib` folder under DevOps Python Tools. ```shell unzip devops-python-tools-master.zip @@ -450,7 +450,7 @@ make update This will git pull and then git submodule update which is necessary to pick up corresponding library updates. -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run ```make update-no-recompile``` (will miss new library dependencies - do full ```make update``` if you encounter issues). +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter issues). ### Testing @@ -476,9 +476,9 @@ You might also be interested in the following really nice Jupyter notebook for H -### Stargazers over time +## Star History -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) +[![Star History Chart](https://api.star-history.com/svg?repos=HariSekhon/DevOps-Python-tools&type=Date)](https://star-history.com/#HariSekhon/DevOps-Python-tools&Date) [git.io/python-tools](https://git.io/python-tools) From 73ff2ef8fe71271639bea252a671c6a70fbf5830 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:32:22 +0200 Subject: [PATCH 263/333] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 955eb8cb6..8c3ad36f1 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,7 @@ Environment variables are supported for convenience and also to hide credentials - [Hadoop](http://hadoop.apache.org/) HDFS: - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing `hdfs dfs -cat | md5sum`) - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output From 94d58f49cb50bbe8ef645d3a785914cd6231b6ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 20:48:32 +0200 Subject: [PATCH 264/333] updated README.md --- README.md | 253 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 181 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 8c3ad36f1..f9fdde42d 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,8 @@ cd pytools make ``` -To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension instead of `.py`: +To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension +instead of `.py`: ```shell make anonymize.pyc @@ -165,12 +166,15 @@ Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://git All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list +eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and +often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. ### DevOps Python Tools - Inventory - Linux: - - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing + - lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -181,27 +185,57 @@ Environment variables are supported for convenience and also to hide credentials - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) - - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes + communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support + teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel + before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation + important for anonymization rules, as well as maintaining file content order. On servers this parallelization can + result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file + basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, + useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). + Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See + further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in + your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [Amazon Web Services](https://aws.amazon.com/): - - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally + filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have + never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. + Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for + users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to + [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is + triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and + [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP + address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its + accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via + VPC connector routing + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with + their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` - - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all docker images eg. `docker_pull_all_images_all_tags.sh` + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a + docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very + useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only + the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests + across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official + `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned + to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy + shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. + `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all + docker images eg. `docker_pull_all_images_all_tags.sh` - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` - [Spark](https://spark.apache.org/) & Data Format Converters: - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter @@ -213,7 +247,13 @@ Environment variables are supported for convenience and also to hide credentials - `xml_to_json.py` - XML to JSON converter - `json_to_xml.py` - JSON to XML converter - `json_to_yaml.py` - JSON to YAML converter - - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for + pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and + [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / + directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard + output for convenient command line chaining and redirection, optionally continues on error, collects broken records + to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not + technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) - see also `validate_*.py` further down for all these formats and more - [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: @@ -223,69 +263,119 @@ Environment variables are supported for convenience and also to hide credentials - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control + - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for + proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable using this tool + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable + using this tool - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing `hdfs dfs -cat | md5sum`) - - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to + replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file + or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data + layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster + than doing `hdfs dfs -cat | md5sum`) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree + showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format + to standard output The following programs can all optionally filter by database / table name regex: - - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala table - - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation between cluster migrations - - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for finding unusually wide tables - - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. Useful for finding unusually big tables - - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL property from describe table + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala + table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation + between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for + finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. + Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where + any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or + subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. + Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain + NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL + property from describe table - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting tables - - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, + with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI + tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting + tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, + giving per region row counts and % of total rows for the table as well as median and quartile row counts per + regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in + an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row + counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating + on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an + optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or + memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average + since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / + total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or + average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, + write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular + RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received + the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew + and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase + tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - `pig_udfs.jy` - Pig Jython UDFs for Hadoop -- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, + useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex + content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for + which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you + want to query cluster wide information available from any online peer (eg. Elasticsearch) + - The following are simplified specialisations of the above program, just pass host arguments, all the details have + been baked in, no switches required - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run + multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server + (run multiple of these for load balancing) - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, + Catalog or Statestore - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) + - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the + [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - + useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks + session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot + debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as + Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: @@ -297,14 +387,20 @@ Environment variables are supported for convenience and also to hide credentials - Parquet - XML - YAML - - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) - - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies + - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, + `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) + - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for + things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images + [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & + Linux technologies ### Detailed Build Instructions #### Python VirtualEnv localized installs -The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages instead of installing from PyPI then follow the Manual Build section below. +The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it +detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages +instead of installing from PyPI then follow the Manual Build section below. ### Manual Setup @@ -337,7 +433,8 @@ mv -v pylib-master pylib mv -vf pylib pytools/ ``` -Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). +Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal +mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files. @@ -355,9 +452,11 @@ sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt #### Mac OS X -The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing `git` is enough to trigger Xcode install). +The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing +`git` is enough to trigger Xcode install). -I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): +I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for +development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): ```shell bash-tools/install/install_homebrew.sh @@ -373,7 +472,8 @@ If failing to build an OpenSSL lib dependency, just prefix the build command lik sudo OPENSSL_INCLUDE=/usr/local/opt/openssl/include OPENSSL_LIB=/usr/local/opt/openssl/lib ... ``` -You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is caused by pip 10 vs pip 9 and downgrading will work around it: +You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is +caused by pip 10 vs pip 9 and downgrading will work around it: ```shell sudo pip install --upgrade pip==9.0.1 @@ -398,7 +498,8 @@ Run like so: jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` -The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. +The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop +APIs. See below for procedure to install Jython if you don't already have it. @@ -422,7 +523,10 @@ Then add the Jython install bin directory to the $PATH or specify the full path ### Configuration for Strict Domain / FQDN validation -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my +[PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, +`.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal +environments). ### Python SSL certificate verification problems @@ -433,7 +537,8 @@ If you end up with an error like: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:765) ``` -It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One quick fix is to do the following: +It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One +quick fix is to do the following: ```shell sudo pip uninstall -y certifi && @@ -450,7 +555,9 @@ make update This will git pull and then git submodule update which is necessary to pick up corresponding library updates. -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter issues). +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each +time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter +issues). ### Testing @@ -466,13 +573,15 @@ To trigger all tests run: make test ``` -which will start with the underlying libraries, then move on to top level integration tests and functional tests using docker containers if docker is available. +which will start with the underlying libraries, then move on to top level integration tests and functional tests using +docker containers if docker is available. ### Contributions Patches, improvements and even general feedback are welcome in the form of GitHub pull requests and issue tickets. -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another +Hortonworks guy Jonas Straub: From 68cdf7de64f0e0e6193c99ad330dcd4e92e6b4f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Sep 2024 01:17:17 +0200 Subject: [PATCH 265/333] removed .markdownlint.rb --- .markdownlint.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .markdownlint.rb diff --git a/.markdownlint.rb b/.markdownlint.rb deleted file mode 100644 index e69de29bb..000000000 From 2ff0c6b46c0cb830261565739e20e365dad717e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 18:40:41 +0100 Subject: [PATCH 266/333] updated azure-pipelines.yml --- azure-pipelines.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e26e7f760..b430c58dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,19 +22,26 @@ trigger: - master +variables: + # ubuntu version + os_version: '22.04' + pool: # there is no /dev/stderr on this azure build! #vmImage: 'ubuntu-latest' - # Ubuntu 16.04 required for docker container support, looks like 18.04 works too - vmImage: 'ubuntu-22.04' + #vmImage: 'ubuntu-22.04' + vmImage: 'ubuntu-$(os_version)' # unprivileged container without sudo, cannot install dependencies -#container: ubuntu:18.04 +#container: ubuntu:22.04 steps: + - script: cat /etc/*-release + displayName: OS Release + # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' - script: env | sort - displayName: env + displayName: Environment # doesn't work in container due to unprivileged execution and lack of sudo #- script: sudo apt-get update && sudo apt-get install -y git make @@ -50,7 +57,7 @@ steps: # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( - script: | - sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c ' + sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c ' set -ex cd /code setup/ci_bootstrap.sh From def50a98abb78cca6c938bb9fcca1ac660fde7a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 267/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e7c73c3c4..399bc52b2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e7c73c3c436c41901d1b51c872e5c3e2b22e2b07 +Subproject commit 399bc52b2b9fa0d3aa4b60d2eb18cb1e3b3ae0b6 From 38c02005e8cd21b410901588698693ea6f26eebf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 268/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 092819070..81714c62c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 092819070eda268f0a4c6f8b373290c09a9de421 +Subproject commit 81714c62ccfee06feef0ab8cef5c1485bff7d9c0 From 566d308bd84fa3a7ff327d7f05f67a83bb05c0ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 269/333] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ace6d6400..575080b54 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ace6d6400c1f647f2e96bd6dc9b730cf1232847a +Subproject commit 575080b548fec8b2420520e161973a861cbbce30 From fbca030c08ef4b3af3a131252dfbef8459bdd031 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 270/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index b8d565217..da9d4c1ab 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit b8d565217a111514a2e061cec896f258712985fb +Subproject commit da9d4c1ab5787bbd3802db78a2330ad713ee3cbe From b77d999a26ca7f335eccc397881d3981536d8dc0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:26 +0100 Subject: [PATCH 271/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 399bc52b2..4c661ca88 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 399bc52b2b9fa0d3aa4b60d2eb18cb1e3b3ae0b6 +Subproject commit 4c661ca889cbef24e989f27c184d12d02cb99c98 From c6ab1cc4353d5a55bff3b41a775635537a9c5f7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:26 +0100 Subject: [PATCH 272/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 81714c62c..666501ada 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 81714c62ccfee06feef0ab8cef5c1485bff7d9c0 +Subproject commit 666501adaf43473a0c94b2ace714598a80cdb549 From 1981197b571c7f661bc912648878f6ef46cf37ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:27 +0100 Subject: [PATCH 273/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index da9d4c1ab..6cee5eb76 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit da9d4c1ab5787bbd3802db78a2330ad713ee3cbe +Subproject commit 6cee5eb764acd9f9198bd83f9af3b17bf49e1283 From 3e40bd427b4780b74718a7dbdf395cf33484a814 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:12 +0100 Subject: [PATCH 274/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4c661ca88..3dd58df8c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4c661ca889cbef24e989f27c184d12d02cb99c98 +Subproject commit 3dd58df8cbaeaf22661c2cf68f289535508c4685 From 5b7e885adc73fada6c37497d18fabc7c1290c47f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:13 +0100 Subject: [PATCH 275/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 666501ada..c90dd3e3a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 666501adaf43473a0c94b2ace714598a80cdb549 +Subproject commit c90dd3e3ae1ccb64ed0e668750fa594b3c709563 From bff7195cde897972a547360784d5a0f7129fd559 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:13 +0100 Subject: [PATCH 276/333] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6cee5eb76..2d12574ce 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6cee5eb764acd9f9198bd83f9af3b17bf49e1283 +Subproject commit 2d12574ceafbcb13f559dfa46693024616b8e619 From c1a5672830bd8eb9628f505a305186cbbd65100c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:34:26 +0100 Subject: [PATCH 277/333] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f377f523f..80ff670d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ GitPython==2.1.15 #happybase==1.0.0 humanize==0.5.1 -impyla==0.16.0 +impyla==0.19.0 jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 From 5c900849a8d38f989a1343c3c36d54d7026adb03 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:38:31 +0100 Subject: [PATCH 278/333] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3dd58df8c..adc18db20 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3dd58df8cbaeaf22661c2cf68f289535508c4685 +Subproject commit adc18db20909106dbf4d6ededa1c6c66d5c28e8c From f373795d24110882a5e8a036484dfa0d5a423ded Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:38:31 +0100 Subject: [PATCH 279/333] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c90dd3e3a..eb10979e3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c90dd3e3ae1ccb64ed0e668750fa594b3c709563 +Subproject commit eb10979e3572ef37b35bda3653b25d7a45e25169 From 2e739a43d79ff855a58c1d76725065c7d1257d34 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 22:12:45 +0100 Subject: [PATCH 280/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index adc18db20..6fc486cfb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit adc18db20909106dbf4d6ededa1c6c66d5c28e8c +Subproject commit 6fc486cfb85886c8b125c3649b80de1762eaba21 From 84e699a8bdae550158a4940234c98996666efbbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Sep 2024 11:52:18 +0100 Subject: [PATCH 281/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6fc486cfb..176f383c9 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6fc486cfb85886c8b125c3649b80de1762eaba21 +Subproject commit 176f383c9148361e04dfde142054ce2efbdbb5e6 From b580060646cdba0f99af7e8bb557a53491d223a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Sep 2024 12:22:58 +0100 Subject: [PATCH 282/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 176f383c9..34f300de0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 176f383c9148361e04dfde142054ce2efbdbb5e6 +Subproject commit 34f300de0e37f5cba5176796560c99e3780a7cc1 From c9f8ba3236600b8666053f3c9fc7bdbcfeff7dd2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Sep 2024 00:07:20 +0100 Subject: [PATCH 283/333] updated .envrc --- .envrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.envrc b/.envrc index 7555ecba9..bafe8df0b 100644 --- a/.envrc +++ b/.envrc @@ -201,3 +201,5 @@ fi echo # read .env too #dotenv + +load_if_exists .envrc.local From 1b5dcec3b4c4ca4373615075b6db22c589a4d870 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2024 09:27:46 +0300 Subject: [PATCH 284/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 34f300de0..4f4be21ed 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 34f300de0e37f5cba5176796560c99e3780a7cc1 +Subproject commit 4f4be21ed3006a756971134afa71c283388b111d From ee32cb23b8c4617015737f9c4938c852c0c03f19 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Oct 2024 18:35:23 +0300 Subject: [PATCH 285/333] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e9c1c291..f615a7725 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,14 +30,14 @@ repos: hooks: - id: check-yaml # Common errors - - id: end-of-file-fixer + #- id: end-of-file-fixer # ruins .gitignore Icon\r - id: trailing-whitespace args: [--markdown-linebreak-ext=md] # Git style - id: check-added-large-files - id: check-merge-conflict - id: check-vcs-permalinks - - id: forbid-new-submodules + #- id: forbid-new-submodules # Cross platform - id: check-case-conflict - id: mixed-line-ending From f04002186cf34e64c7a7503d91144fa7521df597 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 11 Oct 2024 17:51:20 +0300 Subject: [PATCH 286/333] added markdown.yaml --- .github/workflows/markdown.yaml | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/markdown.yaml diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml new file mode 100644 index 000000000..cb5395963 --- /dev/null +++ b/.github/workflows/markdown.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# M a r k D o w n +# ============================================================================ # + +--- +name: Markdown + +on: + push: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + Markdown: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Markdown + uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master From 0e7f2891dca92120dc47009e1e496a871b8ac3cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 11 Oct 2024 18:00:17 +0300 Subject: [PATCH 287/333] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f9fdde42d..6d11affff 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ [![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) +[![Markdown](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) From ac683274c27147af43de231a903b2e6945bd63b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:29:36 +0400 Subject: [PATCH 288/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4f4be21ed..956f47624 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4f4be21ed3006a756971134afa71c283388b111d +Subproject commit 956f47624a5fcefdf46c5d675298e0526121713c From b9a12fcc4a414016b44da2af0cd060f5b97dbe57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:32:27 +0400 Subject: [PATCH 289/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 956f47624..cf080c2c5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 956f47624a5fcefdf46c5d675298e0526121713c +Subproject commit cf080c2c584de729b3bbbcefb3cfa353cf64216a From 5206c0ff64ecd8f3b08d050859be1b6080011631 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:47:40 +0400 Subject: [PATCH 290/333] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f615a7725..6ae7e968c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,10 +46,10 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] - - repo: https://github.com/psf/black - rev: 24.8.0 - hooks: - - id: black + #- repo: https://github.com/psf/black + # rev: 24.8.0 + # hooks: + # - id: black # Git secrets Leaks - repo: https://github.com/awslabs/git-secrets.git From ea2ed7d241ea06422fea231a2fcc513b17597ab6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:47:41 +0400 Subject: [PATCH 291/333] permitted space in section headers to allow for AWS [profile blah] sections --- validate_ini.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_ini.py b/validate_ini.py index 54a01cc63..b38b3ab6a 100755 --- a/validate_ini.py +++ b/validate_ini.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.12.2' +__version__ = '0.12.3' class IniValidatorTool(CLI): @@ -72,7 +72,7 @@ def __init__(self): self.re_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I) # In Windows ini key cannot contain equals sign = or semicolon ; # key=val or [section] - self.re_ini_section = re.compile(r'^\s*\[([\w=\:\.-]+)\]\s*$') + self.re_ini_section = re.compile(r'^\s*\[([\w\s=\:\.-]+)\]\s*$') self.re_ini_key = re.compile(r'^\s*(?:[^\[;=]+)s*$') # INI value can be anything .* so not regex'ing it self.valid_ini_msg = ' => INI OK' From 67c680b35148ee03a249575ed54a0c047ead4144 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:52:07 +0400 Subject: [PATCH 292/333] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ae7e968c..787fb79ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,6 +46,7 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] + # rewrites python files with useless changes like changing single quotes to double quotes #- repo: https://github.com/psf/black # rev: 24.8.0 # hooks: From 510fd69c3f7c7ee0a98ad81ae26123c8f7fb1be9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 24 Nov 2024 04:06:10 +0400 Subject: [PATCH 293/333] updated test_dockerhub_search.sh --- tests/test_dockerhub_search.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index 92e3d2e9d..2a4f9c48e 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -36,7 +36,7 @@ check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED # shellcheck disable=SC2016 -check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" +check '[ "$(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep -c "^harisekhon/[A-Za-z0-9_-]*$")" = 40 ]' "DockerHub Search quiet mode for shell scripting" echo echo From 22fa4cff9849d3230cbbb48a1733c6403286510b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Dec 2024 00:03:50 +0700 Subject: [PATCH 294/333] updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09026fc6a..ab6c6499d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,7 +37,7 @@ workflows: jobs: build: docker: - - image: cimg/base:2021.04 + - image: cimg/base:2024.12 resource_class: small steps: - checkout From 6b56bd27261b489177dece866bc49ca7fdd2364a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 2 Feb 2025 00:21:59 +0700 Subject: [PATCH 295/333] added lint exception for on: truthy --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/codeowners.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_11.yaml | 2 +- .github/workflows/debian_12.yaml | 2 +- .github/workflows/docker_pytools_alpine.yaml | 2 +- .github/workflows/docker_pytools_debian.yaml | 2 +- .github/workflows/docker_pytools_fedora.yaml | 2 +- .github/workflows/docker_pytools_ubuntu.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/grype.yaml | 2 +- .github/workflows/json.yaml | 2 +- .github/workflows/kics.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_11.yaml | 2 +- .github/workflows/mac_12.yaml | 2 +- .github/workflows/markdown.yaml | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.11.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/shellcheck.yaml | 2 +- .github/workflows/trivy.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_22.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/validate.yaml | 2 +- .github/workflows/xml.yaml | 2 +- .github/workflows/yaml.yaml | 2 +- 38 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a678d6592..dc5dfa86c 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -14,7 +14,7 @@ --- name: Alpine -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index fd88e54cb..a522558aa 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -14,7 +14,7 @@ --- name: Alpine 3 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 8f31b2672..209b3cef1 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -22,7 +22,7 @@ --- name: Checkov -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 4dc3a09c3..6a0edd459 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -18,7 +18,7 @@ --- name: CodeOwners -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d56453263..7794a2914 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -14,7 +14,7 @@ --- name: Debian -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 735603e28..e8fec2ae1 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -14,7 +14,7 @@ --- name: Debian 10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml index de3b4b96f..3bebff0c9 100644 --- a/.github/workflows/debian_11.yaml +++ b/.github/workflows/debian_11.yaml @@ -14,7 +14,7 @@ --- name: Debian 11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml index d543710b1..3dbcdd303 100644 --- a/.github/workflows/debian_12.yaml +++ b/.github/workflows/debian_12.yaml @@ -14,7 +14,7 @@ --- name: Debian 12 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index 769f90e49..7119d6852 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Alpine) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index b2b718ac0..24f8f21dd 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Debian) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 70c545db6..ad1a5ed8d 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Fedora) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index e69018c31..884917919 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Ubuntu) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 0975be19d..3dfae11a4 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -14,7 +14,7 @@ --- name: Fedora -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index c7ff16fd4..3c1aecadf 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -20,7 +20,7 @@ --- name: Fork Sync -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 94398083f..bedec7659 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,7 +22,7 @@ --- name: Fork Update PR -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml index 6a321f86e..351180e94 100644 --- a/.github/workflows/grype.yaml +++ b/.github/workflows/grype.yaml @@ -18,7 +18,7 @@ --- name: Grype -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index f68e309e5..83fa5ef83 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -20,7 +20,7 @@ --- name: JSON -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index 033f4331e..f7f7383fe 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -18,7 +18,7 @@ --- name: Kics -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index ac6957e87..4959ef720 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -14,7 +14,7 @@ --- name: Mac -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml index 2d948e136..64ec035b6 100644 --- a/.github/workflows/mac_11.yaml +++ b/.github/workflows/mac_11.yaml @@ -14,7 +14,7 @@ --- name: Mac 11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml index 7d2bf2dc6..5fa32d10c 100644 --- a/.github/workflows/mac_12.yaml +++ b/.github/workflows/mac_12.yaml @@ -14,7 +14,7 @@ --- name: Mac 12 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml index cb5395963..72e301ba0 100644 --- a/.github/workflows/markdown.yaml +++ b/.github/workflows/markdown.yaml @@ -18,7 +18,7 @@ --- name: Markdown -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index de6f79440..7d953dfd7 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -14,7 +14,7 @@ --- name: Python 3.10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml index 8e2d041ac..91658c8e9 100644 --- a/.github/workflows/python3.11.yaml +++ b/.github/workflows/python3.11.yaml @@ -14,7 +14,7 @@ --- name: Python 3.11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 58ad6c210..04c7146c7 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -14,7 +14,7 @@ --- name: Python 3.7 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index f1c806dc2..37ad93654 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -14,7 +14,7 @@ --- name: Python 3.8 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 31c7d0ab8..9cf18e290 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -14,7 +14,7 @@ --- name: Python 3.9 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 43b3f5bd2..3ac4c40b9 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -20,7 +20,7 @@ --- name: Semgrep Cloud -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index f1e452f88..94fd43a22 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -22,7 +22,7 @@ --- name: Semgrep -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 753a6a495..ea4a967eb 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -20,7 +20,7 @@ --- name: ShellCheck -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 26f01fa94..c31ec991e 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -20,7 +20,7 @@ --- name: Trivy -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 0d7fd952c..8edea5c2d 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 5145177bb..ae5ec97ef 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu 20.04 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml index b9bf17a95..d5dfb94a8 100644 --- a/.github/workflows/ubuntu_22.04.yaml +++ b/.github/workflows/ubuntu_22.04.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu 22.04 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 53307dd73..2efa3ac6d 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -14,7 +14,7 @@ --- name: GitHub Actions Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 94a8c007a..468d875e3 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -20,7 +20,7 @@ --- name: Validation -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml index 8b07ed256..3f6265a6a 100644 --- a/.github/workflows/xml.yaml +++ b/.github/workflows/xml.yaml @@ -20,7 +20,7 @@ --- name: XML -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index d252f67d2..5c3118c33 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -20,7 +20,7 @@ --- name: YAML -on: +on: # yamllint disable-line rule:truthy push: branches: - master From 7738761d337f64222906c6a623ed410dfb18fa44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 2 Feb 2025 00:23:39 +0700 Subject: [PATCH 296/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cf080c2c5..a97248860 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cf080c2c584de729b3bbbcefb3cfa353cf64216a +Subproject commit a97248860172c6bb3534d08ad38c4cfdfd09c306 From 406fce6a86160038a5ca122f54d0a401fab1e93b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2025 20:35:59 +0700 Subject: [PATCH 297/333] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d11affff..22741ad41 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) +[![LineCount](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2)](https://github.com/boyter/scc/) +[![Cocomo](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2&category=cocomo)](https://github.com/boyter/scc/) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) From c9f6bce6e1972482bd39a67a793c6c241398933a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2025 14:00:14 +0700 Subject: [PATCH 298/333] changed concurrency lock to be workflow-ref format --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml.disabled | 2 +- .github/workflows/centos7.yaml.disabled | 2 +- .github/workflows/centos8.yaml.disabled | 2 +- .github/workflows/codeowners.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_11.yaml | 2 +- .github/workflows/debian_12.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml.disabled | 2 +- .github/workflows/debian_9.yaml.disabled | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/grype.yaml | 2 +- .github/workflows/kics.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_11.yaml | 2 +- .github/workflows/mac_12.yaml | 2 +- .github/workflows/pypy2.yaml.disabled | 2 +- .github/workflows/pypy3.yaml.disabled | 2 +- .github/workflows/python2.7.yaml.disabled | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.11.yaml | 2 +- .github/workflows/python3.6.yaml.disabled | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/shellcheck.yaml | 2 +- .github/workflows/trivy.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml.disabled | 2 +- .github/workflows/ubuntu_16.04.yaml.disabled | 2 +- .github/workflows/ubuntu_18.04.yaml.disabled | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_22.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/validate.yaml | 2 +- .github/workflows/yaml.yaml | 2 +- 44 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index dc5dfa86c..9bed93efb 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index a522558aa..f6c4f5ea9 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos.yaml.disabled b/.github/workflows/centos.yaml.disabled index ef6aace7b..420b88392 100644 --- a/.github/workflows/centos.yaml.disabled +++ b/.github/workflows/centos.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos7.yaml.disabled b/.github/workflows/centos7.yaml.disabled index a891a3dbb..ec764d288 100644 --- a/.github/workflows/centos7.yaml.disabled +++ b/.github/workflows/centos7.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos8.yaml.disabled b/.github/workflows/centos8.yaml.disabled index d39fc2b0a..76c476664 100644 --- a/.github/workflows/centos8.yaml.disabled +++ b/.github/workflows/centos8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 6a0edd459..168d4914c 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 7794a2914..d84cb16b1 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index e8fec2ae1..e857f280f 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml index 3bebff0c9..ad765f1ad 100644 --- a/.github/workflows/debian_11.yaml +++ b/.github/workflows/debian_11.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml index 3dbcdd303..dfbc4ec97 100644 --- a/.github/workflows/debian_12.yaml +++ b/.github/workflows/debian_12.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 1cb4e5431..f3062b6f8 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index e987d7e46..1b0e00295 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_8.yaml.disabled b/.github/workflows/debian_8.yaml.disabled index 37640f59e..b41ead7d3 100644 --- a/.github/workflows/debian_8.yaml.disabled +++ b/.github/workflows/debian_8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_9.yaml.disabled b/.github/workflows/debian_9.yaml.disabled index 42e28702b..451348091 100644 --- a/.github/workflows/debian_9.yaml.disabled +++ b/.github/workflows/debian_9.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 3dfae11a4..5a4f9a9b4 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 3c1aecadf..6b228ca33 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -34,7 +34,7 @@ permissions: contents: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index bedec7659..c74fa2a0b 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -37,7 +37,7 @@ permissions: pull-requests: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml index 351180e94..1b0fa15b4 100644 --- a/.github/workflows/grype.yaml +++ b/.github/workflows/grype.yaml @@ -46,7 +46,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index f7f7383fe..c389f5df0 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -46,7 +46,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 4959ef720..438dd184e 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml index 64ec035b6..cc392b105 100644 --- a/.github/workflows/mac_11.yaml +++ b/.github/workflows/mac_11.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml index 5fa32d10c..0bcd6e6fd 100644 --- a/.github/workflows/mac_12.yaml +++ b/.github/workflows/mac_12.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pypy2.yaml.disabled b/.github/workflows/pypy2.yaml.disabled index 7dbc25c7f..01080bcc8 100644 --- a/.github/workflows/pypy2.yaml.disabled +++ b/.github/workflows/pypy2.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pypy3.yaml.disabled b/.github/workflows/pypy3.yaml.disabled index ac669813c..101b36751 100644 --- a/.github/workflows/pypy3.yaml.disabled +++ b/.github/workflows/pypy3.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python2.7.yaml.disabled b/.github/workflows/python2.7.yaml.disabled index ff3e51dd1..1813931e7 100644 --- a/.github/workflows/python2.7.yaml.disabled +++ b/.github/workflows/python2.7.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 7d953dfd7..8ec3008b9 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml index 91658c8e9..0f27b0076 100644 --- a/.github/workflows/python3.11.yaml +++ b/.github/workflows/python3.11.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.6.yaml.disabled b/.github/workflows/python3.6.yaml.disabled index 84dce85e0..91bd3e673 100644 --- a/.github/workflows/python3.6.yaml.disabled +++ b/.github/workflows/python3.6.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 04c7146c7..73a0045c7 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 37ad93654..abcbc4c68 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 9cf18e290..1ce7eb238 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 3ac4c40b9..c2f614222 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 94fd43a22..04a3cf197 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -50,7 +50,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index ea4a967eb..246087bd4 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index c31ec991e..95b2571ca 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -48,7 +48,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 8edea5c2d..fe863f875 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_14.04.yaml.disabled b/.github/workflows/ubuntu_14.04.yaml.disabled index 395184d7b..1442d675d 100644 --- a/.github/workflows/ubuntu_14.04.yaml.disabled +++ b/.github/workflows/ubuntu_14.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_16.04.yaml.disabled b/.github/workflows/ubuntu_16.04.yaml.disabled index 0a55dd214..a9655835d 100644 --- a/.github/workflows/ubuntu_16.04.yaml.disabled +++ b/.github/workflows/ubuntu_16.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml.disabled index d909f9529..4b510ba1e 100644 --- a/.github/workflows/ubuntu_18.04.yaml.disabled +++ b/.github/workflows/ubuntu_18.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index ae5ec97ef..47d0512ac 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml index d5dfb94a8..ba8a549b1 100644 --- a/.github/workflows/ubuntu_22.04.yaml +++ b/.github/workflows/ubuntu_22.04.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 2efa3ac6d..0037e4ba3 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 468d875e3..ab6ab9638 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index 5c3118c33..f2a4ca129 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -50,7 +50,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: From 5dbc2ec8cd49f8990767878568ae893ba7b4dd5b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2025 16:31:03 +0700 Subject: [PATCH 299/333] updated .cirrus.yml --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 6916e2577..471dc55ba 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -23,8 +23,8 @@ container: image: ubuntu:18.04 task: - # doesn't work properly - #skip: "!changesInclude('**/*.md')" + env: + TMPDIR: /var/tmp script: - setup/ci_bootstrap.sh - make init From 10d0ad24c89cededf549e38c49f209234d90825b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:05:48 +0700 Subject: [PATCH 300/333] updated semaphore.yml --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 761ef0697..e2b78c511 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -23,8 +23,8 @@ version: v1.0 name: DevOps-Python-tools agent: machine: - type: e1-standard-2 - os_image: ubuntu1804 + type: e2-standard-2 + os_image: ubuntu2204 execution_time_limit: hours: 3 blocks: From 0adb1596606a13a7e3ee873bacd76953d4c6d39f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:28:56 +0700 Subject: [PATCH 301/333] updated config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab6c6499d..b22b98dbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,8 +41,8 @@ jobs: resource_class: small steps: - checkout - - setup_remote_docker: - version: 20.10.11 + #- setup_remote_docker: + # version: 20.10.11 - run: setup/ci_bootstrap.sh - run: make init - run: make From c10d63286ad086d0fb4e3b403f823a38aa7699a0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:56:31 +0700 Subject: [PATCH 302/333] updated config.yml --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b22b98dbb..312bf1020 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,6 +40,11 @@ jobs: - image: cimg/base:2024.12 resource_class: small steps: + # CLI is too old - config validate breaks in test - install new version to fix + # doesn't work - existing version is too old to update + #- run: circleci update + - run: | + curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash - checkout #- setup_remote_docker: # version: 20.10.11 From d8d48a9fc94d856d27c805b56c2287e727debedf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:04:54 +0700 Subject: [PATCH 303/333] updated semaphore.yml --- .semaphore/semaphore.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e2b78c511..308278eee 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,9 +22,10 @@ version: v1.0 name: DevOps-Python-tools agent: + # https://docs.semaphoreci.com/reference/machine-types machine: - type: e2-standard-2 - os_image: ubuntu2204 + type: e1-standard-2 + os_image: ubuntu2004 execution_time_limit: hours: 3 blocks: From 3204b856ac322c4c2e3c5b30bbf562484dae5fa7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:14:01 +0700 Subject: [PATCH 304/333] updated semaphore.yml --- .semaphore/semaphore.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 308278eee..e8de9f5e5 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,6 +44,9 @@ blocks: prologue: commands: - cache restore + - export DEBIAN_FRONTEND=noninteractive + - sudo apt-get update + - sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From 2895d5f6ecc4012b3eded89bdb4990f5e41c3e92 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:21:10 +0700 Subject: [PATCH 305/333] updated semaphore.yml --- .semaphore/semaphore.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e8de9f5e5..fdb60732e 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,9 +44,14 @@ blocks: prologue: commands: - cache restore - - export DEBIAN_FRONTEND=noninteractive - - sudo apt-get update - - sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server + - sudo rm -f /etc/ssh/sshd_config + #- export DEBIAN_FRONTEND=noninteractive + #- sudo apt-get update + #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections + #- sudo dpkg --configure -a --force-confdef --force-confold + #- sudo apt-get install -y openssh-server -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From 99cf65b7fef1586302c8f0979c4cda90aac1ce43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:32:25 +0700 Subject: [PATCH 306/333] updated semaphore.yml --- .semaphore/semaphore.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fdb60732e..5d5a7ab20 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -45,13 +45,20 @@ blocks: commands: - cache restore # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server - - sudo rm -f /etc/ssh/sshd_config - #- export DEBIAN_FRONTEND=noninteractive - #- sudo apt-get update + # + # causes error: + # + # Not replacing deleted config file /etc/ssh/sshd_config + # + #- sudo rm -f /etc/ssh/sshd_config + - export DEBIAN_FRONTEND=noninteractive + - sudo -E apt-get update + - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew" + - sudo dpkg --configure -a --force-confmiss --force-confnew #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections #- sudo dpkg --configure -a --force-confdef --force-confold - #- sudo apt-get install -y openssh-server -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + - sudo apt-get install -y openssh-server # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From f21e0aba680245a800caf478d4b248c8e3870e62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2025 02:07:53 +0700 Subject: [PATCH 307/333] updated semaphore.yml --- .semaphore/semaphore.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5d5a7ab20..59aa80836 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,7 +22,7 @@ version: v1.0 name: DevOps-Python-tools agent: - # https://docs.semaphoreci.com/reference/machine-types + # https://docs.semaphoreci.com/reference/machine-types#linux machine: type: e1-standard-2 os_image: ubuntu2004 @@ -85,9 +85,10 @@ blocks: #- name: DEBUG # value: "1" agent: + # https://docs.semaphoreci.com/reference/machine-types#macos machine: type: a1-standard-4 - os_image: macos-xcode12 + os_image: macos-xcode15 prologue: commands: - cache restore From 1ad7d99584197c2cc80e91d03e6dc335fa2606f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2025 21:38:06 +0700 Subject: [PATCH 308/333] updated .envrc --- .envrc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index bafe8df0b..e4488d51f 100644 --- a/.envrc +++ b/.envrc @@ -71,7 +71,16 @@ fi if [ -f .pre-commit-config.yaml ] && type -P pre-commit &>/dev/null && git rev-parse --is-inside-work-tree &>/dev/null; then - if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" + if [ -L "$hook" ]; then + echo "Detected symlink hook: " + echo + ls -l "$hook" + echo + echo "Removing" + rm -f "$hook" + fi + if ! [ -f "$hook" ]; then echo echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." echo From 6b1938824a0bf6ef7a278c2f08061801dbae7ad9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:41:33 +0800 Subject: [PATCH 309/333] updated bootstrap.sh --- setup/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 94b10159c..d7bedf92a 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -45,11 +45,11 @@ elif [ "$(uname -s)" = Linux ]; then if [ -n "${CI:-}" ]; then export DEBIAN_FRONTEND=noninteractive fi - opts="" + opts="-o DPkg::Lock::Timeout=1200" if [ -z "${PS1:-}" ]; then - opts="-qq" + opts="$opts -qq" fi - $sudo apt-get update $opts + $sudo apt-get update $opts $sudo apt-get install $opts -y git make curl wget --no-install-recommends elif type yum >/dev/null 2>&1; then if grep -qi 'NAME=.*CentOS' /etc/*release; then From 2efb7e7dad0b6979452cc4d229e50e72e32bdf9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:41:34 +0800 Subject: [PATCH 310/333] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index 0b52378ff..b01a77d04 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -72,8 +72,9 @@ elif [ "$(uname -s)" = Linux ]; then retry $sudo apk update retry $sudo apk add --no-progress bash git make elif type apt-get >/dev/null 2>&1; then - retry $sudo apt-get update -q - retry $sudo apt-get install -qy git make + opts="-q -o DPkg::Lock::Timeout=1200" + retry $sudo apt-get update $opts + retry $sudo apt-get install $opts -y git make elif type yum >/dev/null 2>&1; then #retry $sudo yum makecache retry $sudo yum install -qy git make From b13623f40359ddd19ff96f6c2ae78a56375b3222 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:52:41 +0800 Subject: [PATCH 311/333] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a435bc5a5..2a53008d4 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,7 @@ apk-packages-extra: .PHONY: apt-packages-extra apt-packages-extra: - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y default-jdk; fi + if [ -z "$(NOJAVA)" ]; then which java || bash-tools/packages/apt_install_packages.sh default-jdk; fi # for validate_multimedia.py # available in Alpine 2.6, 2.7 and 3.x From 0bba44755a7b874dfc582aa7e15d59c7a94d0986 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:54:28 +0800 Subject: [PATCH 312/333] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2a53008d4..d4434ca84 100644 --- a/Makefile +++ b/Makefile @@ -189,8 +189,8 @@ apk-packages-multimedia: # Debian 9 Stretch onwards, not available in Debian 8 Jessie .PHONY: apt-packages-multimedia apt-packages-multimedia: - $(SUDO) apt-get update - $(SUDO) apt-get install -y --no-install-recommends ffmpeg + $(SUDO) apt-get update -o DPkg::Lock::Timeout=1200 + $(SUDO) apt-get install -o DPkg::Lock::Timeout=1200 -y --no-install-recommends ffmpeg # for validate_multimedia.py .PHONY: yum-packages-multimedia From 72d01f4ef7dc4646eb3a35d665dce0936a445b65 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:55:27 +0800 Subject: [PATCH 313/333] updated Makefile --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d4434ca84..1d98ac524 100644 --- a/Makefile +++ b/Makefile @@ -200,9 +200,10 @@ yum-packages-multimedia: .PHONY: jython jython: - if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi - if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi - if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + @#if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi + @#if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi + @#if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + bash-tools/packages/install_packages.sh wget expect sh jython_install.sh .PHONY: test-lib From feb48b7a21ea137bc18ca92344cdb8a5e75d98bd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:56:01 +0800 Subject: [PATCH 314/333] updated test_validate_multimedia.sh --- tests/test_validate_multimedia.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index c86449aac..3493bae6d 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -32,7 +32,7 @@ if ! type -P ffmpeg &>/dev/null; then echo "WARNING: ffmpeg not installed, skipping validate_multimedia.py tests" exit 0 if type -P apt-get &>/dev/null; then - sudo apt-get install -y ffmpeg + sudo apt-get install -o DPkg::Lock::Timeout=1200 -y ffmpeg elif type -P yum &>/dev/null; then echo "WARNING: cannot auto-install ffmpeg on RHEL/CentOS, the 3rd party repos and deps are seriously broken" fi From 55835218d3367e90d76d6a0614727ede7351e1f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:38:30 +0800 Subject: [PATCH 315/333] added plot_uk_marriage_rates.py --- plot_uk_marriage_rates.py | 158 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 plot_uk_marriage_rates.py diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py new file mode 100755 index 000000000..6b1297417 --- /dev/null +++ b/plot_uk_marriage_rates.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2025-04-26 00:37:28 +0800 (Sat, 26 Apr 2025) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Plots the Marriage rates in England & Wales using the latest download from the UK Government website: + + https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/marriagecohabitationandcivilpartnerships/bulletins/marriagesinenglandandwalesprovisional/2021and2022 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import platform +import subprocess +import sys +#import time +import traceback +import pandas as pd +import matplotlib.pyplot as plt +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +# pylint: disable=too-few-public-methods +class PlotUKMarriageRates(CLI): + + def __init__(self): + # Python 2.x + super(PlotUKMarriageRates, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = 0 + + # def add_options(self): + # super(PlotUKMarriageRates, self).add_options() + # + # def process_options(self): + # super(PlotUKMarriageRates, self).process_options() + + def run(self): + if not self.args: + self.usage("Provide path to datadownload.xlsx") + + file_path = self.args[0] + + if not os.path.isfile(file_path): + self.usage(f"Invalid argument provided, not a file: {file_path}") + + log.info(f"Loading file: {file_path}") + xls = pd.ExcelFile(file_path) + + log.info("Parsing xls") + # Read the relevant sheet and skip the metadata + df = xls.parse('Figure 2', skiprows=6) + + # Set proper column headers + df.columns = df.iloc[1] + df = df[2:] # Drop the header rows + + # Rename columns for clarity + df.columns = ['Year', 'Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women'] + + # Convert data types + df['Year'] = df['Year'].astype(int) + for col in ['Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women']: + df[col] = pd.to_numeric(df[col], errors='coerce') + + log.info("Plotting") + plt.figure(figsize=(10, 6)) + plt.plot(df['Year'], df['Opposite-sex Men'], label='Opposite-sex Men') + plt.plot(df['Year'], df['Opposite-sex Women'], label='Opposite-sex Women') + plt.plot(df['Year'], df['Same-sex Men'], label='Same-sex Men') + plt.plot(df['Year'], df['Same-sex Women'], label='Same-sex Women') + + plt.title('Marriage Rates Over Time (England & Wales)') + plt.xlabel('Year') + plt.ylabel('Marriage Rate per 1,000 People') + plt.legend() + plt.grid(True) + plt.tight_layout() + #plt.show() + # + # doesn't show anything because the script finishes before the GUI event loop + # has time to process and display the window + #plt.show(block=False) + # + # pylint: disable=line-too-long + # + # doesn't work even with this hack: + # + # WARNING: NSWindow geometry should only be modified on the main thread! This will raise an exception in the future + # + #import threading + #threading.Thread(target=plt.show).start() + #sleep_secs = 20 + #log.info(f"Sleeping for {sleep_secs} secs to allow you to see the graph pop-up") + #time.sleep(sleep_secs) + + image_path = os.path.splitext(file_path)[0] + '.png' + log.info("Generating output image: {image_path}") + + #if os.path.exists(image_path): + # log.warning(f"Image page already exists, skipping recreating for safety: {image_path}") + # #plt.close() + # return + + # doesn't solve blank png + #plt.gcf().canvas.draw() + + # results in blank png + #plt.savefig(image_path) + + fig = plt.gcf() # draw before saving + fig.canvas.draw() # force rendering + plt.savefig(image_path) + + plt.show() + + if platform.system() == "Darwin": + # fire and forget + # pylint: disable=subprocess-run-check + subprocess.run(['open', image_path]) + + #plt.close() + + +if __name__ == '__main__': + PlotUKMarriageRates().main() From 1a0723e81e31574595e06f5cb7f85038aaf033a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:39:20 +0800 Subject: [PATCH 316/333] updated requirements.txt --- requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.txt b/requirements.txt index 80ff670d0..05d204c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,11 +36,17 @@ ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 +matplotlib==3.7.5 + # Python 3.5+ #numpy==1.17.2 # XXX: install broken on new M1 Mac Python 3.9 #numpy==1.16.5 +# for plot_uk_marriage_rates.py +openpyxl==3.1.5 +pandas==2.0.3 + # requires pg_config to build from source #psycopg2==2.8.4 # XXX: install broken on new M1 Mac Python 3.9 From 53def06b018590b505dc43379c0b27e99d5e1fd2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:40:48 +0800 Subject: [PATCH 317/333] added uk_marriage_rates_2022.xslx --- tests/data/uk_marriage_rates_2022.xslx | Bin 0 -> 15337 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/data/uk_marriage_rates_2022.xslx diff --git a/tests/data/uk_marriage_rates_2022.xslx b/tests/data/uk_marriage_rates_2022.xslx new file mode 100644 index 0000000000000000000000000000000000000000..54efc33218a9d57c8f26797118393a120865a840 GIT binary patch literal 15337 zcmeHugGM3^)Wj2nYxaun~hS!31!ngMxq{K!bpw0so^TVrT1YX6tOA z>S1r@q(|>=V_mBzBhw>@+z$5?TBTu8PaP(yg2W$9UA*oxDxY3>Z~V68xWnzsXOgMV zy36?30q>Yw^Y|{n*M&_4)U0*nV znNfCN*;>ks*Vqzsm~ER^Ruk@fCs!U8Jj_w)7yG08ovGzSZWeb4J<}US99$5``p6+h z6Hf>d_7GuufUKbd&s1<}JWUOwwQMhGeS1Kz&zHs7ypyagNpX?09Ml!6fmOiQ;p^Bp zmBg~cmqbhTZyJi7hi?>CMsAuak*Gws_fV}X*J~7PTo{;<$(X;-ey)qmxF1W-#ivE^ z)H(6|fWAET1OF?-5T??uuHw%0Sr&_(1QmE?M1~7;C5!XG3b|<&LXe~rT;u7h@tpJ& z+$2E(R6ojoFU;@wUYXIh^7J+wG#vV&)2eeIT>&!Y*@ydO4vPhtX2K%xDZy?1IgsjMtc9-IcAxpRBO30d1(Am)VTx11{O zz026jjPyy`K|upZG1FLC;no(Jo4gG>`>LXLQw?v|8lis4+7PU)!6@CX_Tk`wVOs z9y7~nofbe;`M5u1IAsm!`Yy27oqaVO5aVG#3%0>2qZ1ukn*H|Rl9k`d$}P4OKbL1_ zauMvZ^RYDBuX!2f_6KEe0w{`NzqBT%q z68cV;%yQ9fBsVomoKq{E;n% zUSP$%fki%#TDrI#Pg0U@wWIYNBRcMtk?_@F!7f`Fx36Jnk&8>>MLDZmbRT)s$l50U%zGZeT0vMybI9Xkd6rBcWqEr z6K6QAzLls=QKsV3s~!VEV_d8bHsSdO(t`7Soi@I`B6ahar0@ey;kVj45IFi(i;r12 z0DnzPPH68NR5jr;&30ii}w`@+B52?HQDIrqEe)HIe38&FDa@R$8e601?t-;JO zazQqnTn7Az96g(S3~G>5xLov{)*RPxTcxmACIgps#|8PRQKR!;$EV05z_)jLMS91; z(BsJj^Jytw5$6(eZ*&fNN?353_%GUXEOudX8C=F=+gl^lh@{GihbOjQ-asVT7DPtlxjK#|;=vVK378N9#pZe^tW4qL?~~lvEOJMK z@<{j=upYek{dSU;hI&?aYV#GpHcP#1C$!=aG4c9+mh~o7I3L&tvH>m<&RV5-cZenn zen+1Sd1ekj($Z-I-}3_4vJD&Y*Q#yPPz3#P?K}4iTicEt)u%Ji_hL1w6kq)A*T6c2 zuGn~3I6HV)?9W}ev$L?h5=IG6E}R0lEaLlj2L@`i9`1SXkE?<&V<787LJkj!R1S#} zN<02_O|KQarNjZ!pD6?g2+kjC`VR*D>&pJchOLQmRy~aHA*X?l&|CL;WGl_g2S1ph z+HA3pEb#z={jOEBHn-O-*fJ`V3*@v9Fj`2#V49?-8w@Qq=EgX+tz1ET1h_6f^tI08LU1w9ehFDjN(!LvXeJ5O4D=!0^ zTbh$_KtvP78a=;Wgu}}-ZF({e8 z^?@TAj5KuQDn@3pG@qhWAYe%nS&Y#E6rI72UesTZ3wyB{*4p-P&~ZB9KLsy zBtDLS1IxVz9?;?u4LG9gRm#j2%=w799Y zEa~&~L(pKUEOC{Uw0l8+l2Vs0j8X^dU%&0}>t|00&9|J;$D(o3F~kwa3Qs;%u1^OO z5GW5I7N4uGH;u9t^EU|ULz<2ud;@J$UyspEOjKLnS?9Epf;?iCn}Xm%p`G%%R_SY3 z=>qZ+c9%4RGD3JhCFzGDjBFSaCnOP#q;2EeTlw(0Od@gCuR@RY$(el9=pE3XQWj2{ z-&@Us8q?wK>+_*zHh4{obo*xZvc zlSCRwHaxBc=Tb2lGu$Jz=51ali9%zldk2%B2AN@McIcY7CYbsA?bh^Jb16s&$?lF2 z=^@4&VihV?4P#!gKCxy@bKV7hN>J`byw)QoEdd+f!%}8Sm45)eBkt>J#~~e=;<*a60WXaq_t zRqJiF@AqI5bJT%v2{G(8`|G-skZ)856sjKWZ3!h;geYN{YvGukfsnS>$d_h+dUfiH z(*4DQxVMy1YLhb8j&j;hI_#|(Y7v#s5*qb+7V@|>&W=Y`fz;EQW%%h8!!Hy0^%AMQ zWQ$J&|CmHg$cSxddruiY!*_%fULx(oy;~fwfosX=MD_*YEcBZab^p`{DWN6#C&X`*!~ruw?WAJ2yW63F+p!vf&JHJHIq?!O?p{ zw?lmee`cvz2?c3|aG*7T0p~f9^Yd@+>Kr=uLm>(_N==5q`fqvAnX@^DbpZ?#K1zxg6k#r6mT>(wCQtpg~kFo#R3A>XUa+c=n&SA&j* z6zi!FLoC`^F>ngqb5VDf*wdqD`MVxjc+om;%d`TB)^t2wEB$9M?#&6{I>Y&3nEPAw z^qxkW3ruSXB1Q*u#LI(7qjj0lS)mu^+nG?j7v{Sc#hrb6j-GU&POCx;!_ymz?v>y? z@A>*@zB5-vU@@HxJ`%icES8sNP%s?h#qY=*$~`${r!~3nVZg_yOMdN;rL-gVhLN2@ zm_@Vly)AX?xc6`aLadV?abV_X3qHcFbp`Wg*)TP8=Aqgi`8@sy*JYLgIC`Q{>I~Tg z3^$JUauiY;*^9(^*-P$|9cJ!fgj6mb#r!#?j}5h(r~-WMc=RZgvsYNIaCDt)n&PXdp~g5 zFM#HO#rA?@K=G;B_bxA+bY4MF(zoiaT16NAAgpmzV0^OfxjFhSFQ0T?Nl@~s>aIy8 z;}RT2DcvV4H=?Z~yM1qW@DeTd>gtPaY+m$Q6i|kLe4$`LTQbU{%@db6(sRxseTvgAkK>9QqM94Ws@pMo zHnhG6GRN(biR2Z8o?BPs!>yZzWATGV93MK;CLEGOx?jghh{)zPI8I8Oz%#!bVzA$P z-^^M0fP1YU>0wI`NXnH8dbdtzWyvn{LU2Qbv)U`OH)wiUVg`h=a-)Q#S6FER%_kvc zEuGni^hCRl2=XTalK`<+)0~dB=f`1OX8`0yoIqzbjh$oh?H05`vp2gZMgAjoy9r8z z#&l(`kcKhimSrOD3BXXA8|CE12hGvivs1 z_OM1DoX252XEP|43US_)hGAU6mZN2o{JsQX*=YlHV%~J$Ld0g}Kx~bkByO&m8m(tn z_HtVhz4cQRtWo}S-<2%q6`*Lyp{rwYX;b(q+Xv`LCsNzFl{w5J(M1bth`QlUJ5^uYIRp6MSvN zaZoM~7_V_}k^w!m%-xwpBqp)FIDP(-Mei%2f$qD7ukKi^@UUrQOVP+%4z!C7gAEG3 zjjoJqKfupR4?-JSp_1SC-?TF#H{4!Z-4$4I11IIQDqaLI?nf*!q4khQPGVbgs_{cg z!*UW!@Rq2niQr4eemGkqfoU(H+NSd{J<*u%<@+NC4 zCJn`}T1})8lOEqtBXzx9==iDC{D#)N5hJPJDk*trw;x(dHXBQ&?<>b^Il0kC>M{Fj z;pY41lQA(jjf$4{83sk^4@)2c4Ci9F4qn$h&eyjOG`Rx5d7d>&2r)nqUaWHJ2S@-@ z|2r&oTiwP@htQ zrYtD@C~U`66trjz}wI4N1Z6(kcQQOsW1-onv7$Sh_1I-=MN!3=x@zZJF4 zO(vA$TB642g9(Xi>@?@}UwacdAlXftjWK|SAGn%=K~tz)jzt=OqaR?b=;gyqOnQWn zB<<(*xm96q?$apdaLJ)a9Hr3@(Dii;<-^46A!_r*VHv7Dyr|hTk^+U z%mxOGN%c3P04OaH+ni_SCRsyo%}J;8{VvNXBA7sJuR;UxV%>qu!$<9o`l(N#?^pwp zit4{{A12wx8X+j&&E~dfoT`mrtWFr{)imLk#01ahsQOif5W+XFi@{Y2QSs~cgl3w5 zdV4sZgKnVHGcR+juh{UzN>NnGu`z@`xX#k8Y4Npb_g*yGDtw?&TRhdcQY=F0k+O=c zCJ6^XJMx5?ALKolGMUuoP#?U)1k9Jl2uiN?-X-#B7*x9u)HjBWs#FNlYB!bq{!c}u zAL<#I3zGBc0uiTA$hh*y?XmVybh#0c?yq>=AUxaq>`Tf|t?J@Y$R7Wji3r((~ zaxxoZ@cj6QxoXhy^!?zBV}FL?th=}4;dbO>$MYHYW!2{LYle28^Yi@#%r&3OU4NOitfK82KaKpMeTw_A(0h#tWkCxfY*{41($F5FK>8R2_FO;c50m zw(@pZEi@CJHIYT9R*F|cE}KNUV!3gzm!yq+|8AcpyOMCx6y6d(5odKmFI@(5y~~oL zv8lWeRb{q=@BuZvh_h=-rK4IT62G)XBn&Xm$Gj<975?J{xiTU}L zq}z-rXz_e>)uyti*lL!~+mzG@*bheEu+M=0k$3b;N6K8HOlqb zL_I%HUEG<_2FdJjhhE|cb-==JA>I?{^|HsyIPOg~MPRYS@wtABQsi}7l#L(k^?id# zEEeY$D23@Av%n+DCxWN=g=x9rRekrPIY}erwq;r(McAF$7=CqYWNI5BrgK15gxMl* z{kOW&a26i!i_b}mEm$hfk^NX+4g5};6Gki}yG@fCB!)@&N77|9gJWu!8HS){@g-Js zRE{5EXw0t95^bv9+BpQcV^D^ptQFkih^|QMaRM?_Ju{;b<<{B20EF8S(^T`2dyMfd z9a4u;Fj{XvdSaY->)F%z+SLpg8m#giw?kliTVfMkHC&A%D_I5b8il~X!eg!kguQ;! z70oQwwmwNr|CY{<1dcAQ-iDU29o07;{)OVfZJ59f)CmeQM(6B4kvpBzY**C3=G8SJ zt(E7ytmRJo_%f$C4>c$K*_-G467y+&{z|hktJDka{>!UNAFP#CMUlIcA zBwn5kjmCRqe(kL^K^nqz2V`fSL3idZQj3rkETt@YLjMhw3j3#$zBT+ZvIV^EHo~Zw zHgN7;4wM2-O$Hk1=ZkvX~@8(-SOmQVflX6EWy5^;rW_7cL zSA3mRpdrLkSV-&2J*4MNDFMG~Q&{03oE>(8vex^-j5Cjz{SkuLiyFyq%M-}9lSMro;*g%$>0VTS$2#zmZy@2rAC zU@-Hmwu>O`p>*cW-SBejtU0usKd+~~ALl3KJw;)=kEE1$Q@DeP?1mA`30hGwYpq-q zbkDw2*3c``-{=x?j5?521C?`8>Pv%NvD8N%t8@=ssFBtUYss`m49Xkgobt-Hi7A8cHkC_0CXIH+pm+ehqam0&$K{++L|3OE#RA7``ope zi33wb23Ar;rCeB^HY2}S3jU^ghj1_@%z5#-oh3G?c)Gccp^N{X-_LDrg}mtmLc*&A zx5)J1Mk6b*m{xUYo86_I4{)+96h-`nOjjz68+ws*eqjJf$^x)747YF1}?CN2I;wS2%<=Y=sXOv2dvD)hdN_b_>WwRJj_N@zXo zkt|Mo)p%qd0FUa_a_o4;uD+76Gq=@!l=KSEgsedrB_$$gT0GC0b^jK z8k2ngz)mL6t#6Ow^=T~(-B|AW>+r5=xV0F)Y#{8cG`M4y>^sgThAym7L>$#7HS+Bt z@JWZ{^+_%nYH|jqGGdvux3v^_AYfi?in!Upmy4kah2nCgq1ej3Mu{YZGC2)~ zE`FNsg7{obO~FQ{iKH8iL$)os%rO|S*lO+~pIQ_s+@f%#33ZAid5 zMJ^zzo1VDefomNBH$p%Rw~C^QMRM-BC$EBg!@xa8>cb&QI*{;)SZcb6XuB` zvt`EH&>3*^h0MMq+`eyTQzPD zn|G=306^$SNDtABNtH=h%v+rUS$6P8{nt5sfOZ(;(>LdQ9N8Z2xc#OlV_RC=J8^_% zTlOxPfXN`QAN@*uMr(v2CFF#32xy5c9y15U6Em-nfR51}6)q5f?#^xtwf zJOP)92QUHov%>asF}yQ!G&5CkcC@g4`)f@^s>)d{2_m;)-VuiI@g#4HOkqjoChjCd z;DjnlBbj(u(;bC2*Ek;et!Z&t;8IQC1=U{M?zCwoX+T>UD)`sAB~kA*fjLI4y=y3) z={dMNN&t)AV}IA0FO45QtM+&%GK44Oh_6u+Rjp%=%|WNN-E=w_{`N4ik8+@Dq!BI% z*ET$lZGmKboF!tB>YewgaUne%WZQIy=ba-nI%WyI);wUsbO@h)O1FeDnD6tQ_UWBq zm8NiNU-MVxJ!Jh#ZB&T}cqt^0zi&Mp_IChI3 zh55M_OX>4$X71nhcC{q9O}O7P!bDyOrubbKJx@3utO=uW-f0G!9Z=Q*Ih6bh;q zECRxNJEmbXQSfV}cFu#4fzgvEz4G}(-Yz!%X0(W{s?#7N>lP`;ugrB& zOxK5QMs%@j@Fx*>uY|c^co7v2l18;%aO1m0sHKmw*XWO4HF##70D%2CfIa=5_cN|J zVjS&EGg$Dp3Ff=1?_P^^9YWQd2g~TM=y+p|rs><$z~$wo!v|tDU&B=gE0H~7_4xKK zR7b!19LvA6AQkSxICL9)D>UCSU}cS^hMqY=o>r>ymK6zxOEYn$IOCV4&{ zB*hhL$e$<4H7Is`{@z&V(CsSA$_Ejm$mZ+F#nr2S8-YNIyfy=%=6k?K@y7_*1DA@k znWKuCv-8h2!|>uW6TPv8Ev&hu*`SG}k)k1`(I9mEiQtz=;`g~>#AuZI-GVTTUl^-a zjn<8jvqlCx2RnxtBbI_lAqR~OCyY$Nrisw-uw`M=M4Q24>4-q`jI2#ny!CX!u^@;X zfgSkEEd5y_$Azwan**F7L14KM1$c+4or$8OoxKx-iJha_&uuO;Ue0dmXEg!*ju1Zc zaBz4Yiqyg1b?-AM6l-6|3TwDr-Ece+lgs0P;IA%0c5WV`nbafR)wReGC{Y<~+I zT0(%?dg6Tl@!XMxSGJ##mhgUU?P^zAUF+2Fm zKt#ht>r&7T()@&kb~l*%h5Z{;+Wy1|mrf}$XUSRucCg&EY?{^@)Fs1B%xI5^fTgW& z0}W)EwuM@r5ZvsC?5rQs_XZt88|wrJu*EQ_K4k>(%8NCirIj^N#8q_@BGDgguPZ0m z`42=h;35QZ;y=70dfZb4bPHM1pxN$4d=6>1c~<*wXQaW=oxX2tR_yhq_Mm}B7L+^R zC#lCZChoMRrE3R{Tz-jydoKob!PN$K* z{jXX^tenEH+K}W!DCIR9wj^5Gl``{~?B zK#bjbl)azdUE@Z&1&&{-s6H%oQiLI^8VN@H@q~#dv?9XvC6s;wZh7}r^RzUmV@2!E1)(1W~4uDeNw1?@c`(`eW3g!n_}YPa*<#%_rn`H4U0S!jZPDgFUWMTqj3x8bjvlygWyedX8F#T8ezE zV=Zo`g;q=bixJiwm>lh4^gzp&lN-|g)rt`zR!pLU0PK7bOTAoR2q%f}O%*%ZMil+1 zn}%Jr<_0A?EBf^~Q(F&Dvji0HLp{5kNz8e6;Ya>_KE3oM7z!_B6}zPN$40&Mt1j2| zN*lY3!^6Iv^Yi}dPh-|C%eEB_W7f8zFEUj`EN@Uk@yu?`AWX5d=0+cLWgTyf3|XJh4h6aY=zcTiEss%eg0tQbTUn)< z&O^MJWaa>P&A)n|BF1~(izVSjbIwbsVr9I9IhO@18Y`$igp!)ahm%KXZpeqm50JhU zhuLWf&slEuyqI1Br=YP(B1ZPg5$+EffawVt=V;cH;ffGLInt{NDv8Nb0Pik5eDkgT zjJ(Yr&#kEna~h7huI<+OqzUb9ePbhQRIGw?e~@>H-{csCms3wxR#v~lv_mgJ!71Kh z%#s54F>@%s`?K5D#}kbuXxB=)1AXJA!m+SsCp%WBW5By`WNbNIW(gJM3q0s^X0()w z(a)hab{p%H@!aoNNeKvYZy};6wdi*vQaRFdwFv0_2qk2(j8Ka854E%!)6d(I#{m|C z00%XwvQg#18c3o~Tki?T;J2y7{m9;9-|m(~A8oJXA@flJZp~x%goR>1kIpb2l*^?u zyYci=-mYM_7;p)K6l4&kNvay;eq$}H5Qfv5)D4}f^*)Vspc+97m3vCFc+DzTd3*|F$o=zpYa;Yx zFuG-yaH(%7J=>m0@0HF$E*Z%CySSWAXnPV!+Xp!!kt;}&%nHa+6Qft0D#*$R_)I6E zTE2T?OsG8J@oistWA#j#Jj4N zvWgMSjF1{`rBa79p%{epiBTcBw63WOs|=ZNqu2s+!4{AJt3FRUi+Ze@TgLWi?FkA$ zTODB@H?mQVy}?g3|HC=1M~rK@#@$0516K`Upr+|TUWVW49WI!5#eSq?7;kc>GnI>g zM8FOe)a`Y62$Jzz-aI1z8-*Pwz2GDRX`BzRN?N<_DEn$io@{LW&zI{R`bgS1wGW(J zjAL0Pu;tvY{KV)Xo2@CThhnA6R(Q<8VK`&lnaU1H0o1PETrsIQ1O@f^wrljap;>Das{^w*}>IdYuTqXn5 ztHC>;Sv!*;XE<)tQQ$=>#B0jrHRE)gx8=;=M0thUct$>;8x>HEss@nRu7gXB6fdf9 zTWhat+CIL=@22FeJ4!WEpbZ$Xe+TDS@bUFRh=GYmVX38v%lFtN$^;Zo=f-W2<&vGTAr997ILl`33T+c_N|SB<_@ua$ z3g-I-E8f|tpw2An;>f=;%;d}J8_t&)6Yj~dp)8yyFKsxP z`kUa;!qpBNYAvK9)p__zU%l?UlPst)ZxFDq5Am#`#y;HWMm_?zV_-W0w$qE?Q}ROQ zMoFptr|YawmZ}WsmBwneor=Zru&QcY&l#WV4rheHRrNPwU$Y}d*f}gMO~O&}IML5= zUv<2yd;;ck{^=UnLL80t6&X(a}laKq#6)qaXc^29pU+1B<@XlMUOGc$I zZ=pq%RhnNx5is#v6$W_aG#Cvj6hJ^{wMBg%bL;D2q~aBNv-9h> znLe=z(Fw}CTMhJ&9In&n`~owCcDGZlO0#6Vjcs;%B6z0KyhtaO4%d6pCI-#Qg^G2e zi{9vEdf3ax>f9g*aAvC6D#dxIqtHV{OhtnB^Yz!UL-xWVH)_k6PSZcY%MTCc13z-x zEfDCBfBLB4^-&XnYr;*f`*{vcg6iGK4qqpWx=D#bw>lut_K4!uysY(G(+rCi!=sJU zhAkeTI{#Fdz{aY+7I@`k>Y{1FBud^q^^uwgs}J|9#vERK-X6dLreknfvfp2*^D~9lA$%wZ}eJ zR2PxEkxGe1v>vWd?MbEF<#kDNJ|$*b(ejIRwr2S@CM_mQ3v&11N;9gq@%|EH6dcH&Gr0C}j=^@ZSHwe}cxlK7cX7L#?;0^CT{3j=3Yp?YnDX zw1zjX*Eut@^jWB+QQlo)DA3RzF}=EdDp(QdE;~tCP+^@=WuuIY_9T=57uY?W#XygU zYRja`^QT}wY60Z&ZCDr5o^7L*=83X1h8JYc9le21rjf=PG0nj%OJt7W)^e*OJ#f*p zbAGZByf!Kku_Ge9gPOTbl|FxFM+kJg5j(+P9416rT+yV|?e_kf*f_`@LkZPt=-!Z4 zvJBU3bvfX-^0x*4rxoZXQ;_uhFIMo&;Q!AG{$lt*E3gCx$588Dxu=HC1{qh3(&5uG z*xT&laBVmeQnDr^n$i|)zD1ufpm?N(*BIDBy;5coSiC+!v8y4#=I5^tOm^P>xU2^6 zGd8h%`{CWXKYXZbo29RPy>hbqdAOPq32ut$l)jZK8q~;Z@<=mgMMQP!REf;8mPM4y z`U(LTU$z_{7PWcYLA?%87tezDhYq=(b$o&$I?((Fl-!kW;SJQCQ`ztc>x(esPPmY_ zRys*uLJd*A_mzJ?KJaJ2+ZJR@8Vgi#3E2M0^DIAk-a48G5y*Iul>t);bbAwJ1g>PQE_kyN_|oT@U1)D{cncRwK8{B0P==Z-%Xxt~ zBv8=ML4MS7pP6Iwmu0{2C8p+e{a$BL*QKd9g>@9DYQV(T!ZSO=rCT#f935H0bgW0Jdtrz zqI4i{M=xdlo9oFZmcn|KkjruVvKKQvZ)-$ z_T|G5#3DVkT$u};WbTQmbGwnW}p1 z&6OFx-`pP8&8k6OM+(0l&Z==33EMA#=;t+b^dwVXq&2-5tJc$X>=>6w=ru)4?L+Jji0Ak?M1c5kEY9mqj)3l2HW8c zs8x{n0m*=hu&ocmk_`b7dFaj~SacpJ7@>*FZp!EB)Cxn7skfMavz0$N&+7s#|9|7W zU&iu(oCh4gKYxC6p3yJPbNN*uezzU~^cgLt{njlD>CaG^OH0VutHQS1=8iBTlW~Vr z4or(s%V)m0XO`dB6I|BY7`QaC88))aa$-|%&G>WcIm-E3PxC^D+HH@aBs`$yuA~NU zYUr-PnN@tDBj^)Ej+~#9Q8|U=`*C;Du>nkh{y`a_V6eYRQZPWR{`t_?pQ`@x{x6S! zDa!mE;N_D>zX84i*XCcIJ9;Vj^0}Meg4e*P@5as1Lw3pp@Sv&s?AOH-ofB?TM z=`TfJRvLec%HsYZ`m*l$viC1*Prnho@n4R?zgD7N0=_Ke{08(T_yzc)kn^vFotJvP ztiAjOB_aHmO3X{~m%0Am;%7vE5&uu-|E2WHWcY7sM3R>g;jhs5CBVy&`8NO+^-F-? z0rX3hm!ary6ik|zD8D?`mjEwAyWapXv@ZdEb>qKL{%Y=j51W2tfPkF;4dvg0sGmLh zE7sqg@4sT*(EqhZzp;M1;xDy(>B0R*duIGA*3Yo+*XH&T;H3xk8=#ZvCBU!Y`B#*e zzR7QtGUmUS+8@E@Pfz9LnEZWn{w)pya>Vl2Zu}mT-&?h!3?z^MK|nBozw5wbvKMSW GKm8Y$>}z%a literal 0 HcmV?d00001 From add60aa8705f2481287cf0fe9b77c0aa7b4e78fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:42:05 +0800 Subject: [PATCH 318/333] updated plot_uk_marriage_rates.py --- plot_uk_marriage_rates.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py index 6b1297417..a38332443 100755 --- a/plot_uk_marriage_rates.py +++ b/plot_uk_marriage_rates.py @@ -68,10 +68,10 @@ def __init__(self): # super(PlotUKMarriageRates, self).process_options() def run(self): - if not self.args: - self.usage("Provide path to datadownload.xlsx") - - file_path = self.args[0] + file_path = f"{srcdir}/tests/data/uk_marriage_rates_2022.xslx" + if self.args: + file_path = self.args[0] + #self.usage("Provide path to datadownload.xlsx") if not os.path.isfile(file_path): self.usage(f"Invalid argument provided, not a file: {file_path}") From 76d07c239817b67971c1e96aa9246abaf6e7b133 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Nov 2025 23:18:46 +0200 Subject: [PATCH 319/333] updated README.md --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 22741ad41..40045f8f9 100644 --- a/README.md +++ b/README.md @@ -637,24 +637,35 @@ Does nothing: [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) -### DBA - SQL +### Databases - DBA - SQL [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) ### DevOps Reloaded -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) + +### Monitoring + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) ### Templates [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) -### Misc +### Desktop + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) + +### Spotify [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) @@ -664,7 +675,4 @@ The rest of my original source repos are Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). - -![](https://hit.yhype.me/github/profile?user_id=2211051) - From 58f3505733d4fedd2abd8baa6bc69a64cde89580 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 04:09:59 -0500 Subject: [PATCH 320/333] updated README.md --- README.md | 55 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 40045f8f9..e8cfc70d7 100644 --- a/README.md +++ b/README.md @@ -601,8 +601,8 @@ Hortonworks guy Jonas Straub: ### Knowledge -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) +[![Knowledge-Base](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Diagrams-as-Code](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) ### Containerization -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) +[![Kubernetes-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Dockerfiles](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) ### CI/CD -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) +[![GitHub-Actions](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Jenkins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) ### Databases - DBA - SQL -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) +[![SQL-scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) ### DevOps Reloaded -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) +[![HAProxy-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Terraform](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Packer](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Ansible](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Environments](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) ### Monitoring -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) +[![Nagios-Plugins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Nagios-Plugin-Kafka](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Prometheus](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) ### Templates -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) +[![Templates](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Template-repo](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) ### Desktop -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) ### Spotify -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) +[![Spotify-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Spotify-playlists](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) The rest of my original source repos are [here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). -Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/) +and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo. From ba4689ad10b7936ce3b07791671bb0c2890e2b14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 321/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index eb10979e3..d65ec80e0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit eb10979e3572ef37b35bda3653b25d7a45e25169 +Subproject commit d65ec80e001bbf9d15f267079c101ae01578a183 From 572228d476a5cbbd8dd6dcf34217b4d5c3e82738 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 322/333] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 575080b54..fa8985e60 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 575080b548fec8b2420520e161973a861cbbce30 +Subproject commit fa8985e60aaf5bb26f812cdeeb20b676684e3964 From 8c8d338d2cc4b7ef9282ebe9655a38eb31a4963e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 323/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2d12574ce..344a446fc 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2d12574ceafbcb13f559dfa46693024616b8e619 +Subproject commit 344a446fc8588cefe7b812c1fc449a484a27c57b From 351e28284fb9a24d1d187c213c5f090ed99dfce7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:12:41 -0500 Subject: [PATCH 324/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a97248860..22c6dfefe 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a97248860172c6bb3534d08ad38c4cfdfd09c306 +Subproject commit 22c6dfefe61703689e7b09185debd3010d93d9db From 824267aa84016f4cca4b4d6e3b971ef6a91d5a62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:12:41 -0500 Subject: [PATCH 325/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d65ec80e0..9efc29299 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d65ec80e001bbf9d15f267079c101ae01578a183 +Subproject commit 9efc29299c878e8b781d7714eebcd5c06cef7b7d From 58521d67c5239a3516f9fdb08a35734f46e6ddac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 326/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 22c6dfefe..8652100df 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 22c6dfefe61703689e7b09185debd3010d93d9db +Subproject commit 8652100dfb9e07f864dff7b9eb3fb144c3640449 From 40f0b93ba154d7da4c0b2c5f565fe89df6bb9f39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 327/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9efc29299..61ee5978a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9efc29299c878e8b781d7714eebcd5c06cef7b7d +Subproject commit 61ee5978a396f733033af23860014e91969f07e0 From f3cb837c9c2dc10589deaf92c912bccfde093c45 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 328/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 344a446fc..620b30292 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 344a446fc8588cefe7b812c1fc449a484a27c57b +Subproject commit 620b30292fa572083c77159a1d67f4656b759e1b From 555239dc9cf306ddca0f20cfba69f84de122f431 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 04:56:20 -0300 Subject: [PATCH 329/333] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e8cfc70d7..5e1fe9bca 100644 --- a/README.md +++ b/README.md @@ -664,6 +664,7 @@ Does nothing: [![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) [![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![MPV-Scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=MPV-Scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/MPV-Scripts) ### Spotify From a70079597d44a1b0df2cb585dcc3ff96131b73f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:38 -0300 Subject: [PATCH 330/333] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8652100df..11dee29ce 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8652100dfb9e07f864dff7b9eb3fb144c3640449 +Subproject commit 11dee29cea607445270d8bd675d1bcdecb069c14 From 511b9e6f20cbc66a10f6d4ed88a45311552f5916 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:38 -0300 Subject: [PATCH 331/333] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 61ee5978a..0d272f4cf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 61ee5978a396f733033af23860014e91969f07e0 +Subproject commit 0d272f4cf2c27a8ee614eea18c0bd171e47a06cd From 874083a6f3e5ce387d1496b7509c59b30f05aebb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:39 -0300 Subject: [PATCH 332/333] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index fa8985e60..30d80bbd3 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit fa8985e60aaf5bb26f812cdeeb20b676684e3964 +Subproject commit 30d80bbd33f0e6c26c230f26575cb671c41c3e7d From 32ffee202073282e7b7006102e692c453e98b01c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:39 -0300 Subject: [PATCH 333/333] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 620b30292..f59532dd6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 620b30292fa572083c77159a1d67f4656b759e1b +Subproject commit f59532dd67a583be3522814a15024a561b12c5ec