From 14923853f2771751c179cd867a0d94b9ea418bac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 18:44:33 +0100 Subject: [PATCH 001/459] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index eff67422c..00479ecb7 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,7 +22,7 @@ on: required: false default: false schedule: - - cron: '0 10 * * 2' + - cron: '0 10 * * 1' permissions: contents: write From af2e0f141749df2aeac0f488ebbd129453d5ecdc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 002/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index c9878cdf5..4bc47ad4e 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit c9878cdf5cb52eea7366c73967e415a2a25cb62a +Subproject commit 4bc47ad4e3dc0d23f08054b6f4687a92869ec9f5 From 1999b1bbabccf27c89a7ea9c06dfaff42bce782d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 003/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 1dc4ec7c5..b22457e26 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 1dc4ec7c57ea82317910cb6d9a550837af120a68 +Subproject commit b22457e26204158f537524111be4e45669b82663 From 504eb8b6c3a39f38e2b683a9328e988dd4da7a77 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 004/459] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 9ac92d3dc..300fd2a80 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 9ac92d3dcb3c2f72d0b9ab896604e932f5d43aa7 +Subproject commit 300fd2a80741eaf8dcf3e2900f507a5531355bd7 From 749bd6a88cd56ff6c69c59d0aa0e44e282f6b8ec Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 27 Apr 2022 19:01:13 +0100 Subject: [PATCH 005/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 7330e7ab9..9a3f5c616 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 7330e7ab958a08a3f22a70dfdb4d807dc9276d74 +Subproject commit 9a3f5c616667cab76f98c13f2d8576610b877d1f From ed6feacd0214d212eaa0eb96a55fecde0798809b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:27 +0100 Subject: [PATCH 006/459] updated pypy2.yaml --- .github/workflows/pypy2.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml index ff7ebfc1f..7dbc25c7f 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 3e3a05b93f242a81ae2943d73087e9ff2085e529 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:27 +0100 Subject: [PATCH 007/459] updated pypy3.yaml --- .github/workflows/pypy3.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml index be648b6b7..ac669813c 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 058b98407d0b1b131ba6e9ce01ea2059c5a6ef0c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:27 +0100 Subject: [PATCH 008/459] updated python2.7.yaml --- .github/workflows/python2.7.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 6f6a3cc13..60b31b6bc 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 973c93960cd0793efa8f9dd739c4c5249ae8ea66 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 009/459] updated python3.10.yaml --- .github/workflows/python3.10.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index c7710cc6d..939ae7337 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 6516c90d6b7eb00bc64f96b00fb14acd049f0b4d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 010/459] updated python3.6.yaml --- .github/workflows/python3.6.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index a37f4b538..d6a61ea45 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 6bcf6787f950225bf13bce535cd23accce2a1f62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 011/459] updated python3.7.yaml --- .github/workflows/python3.7.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 8c19509a2..70ca1f828 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From edb68351bd5881970da4e00050cd28a202fb53a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:28 +0100 Subject: [PATCH 012/459] updated python3.8.yaml --- .github/workflows/python3.8.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 025424851..fd6dcefb2 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 11bf3fe84c23f316d212bd93bcde98cbd237bc4a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 15:19:29 +0100 Subject: [PATCH 013/459] updated python3.9.yaml --- .github/workflows/python3.9.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 862e2eccf..e8d331278 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -19,22 +19,14 @@ on: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' pull_request: branches: - master - main - paths-ignore: - - '**/*.md' - - '**/*.pl' - - '**/*.rb' - - '**/*.go' - - '**/*.sh' + paths: + - '**/*.py' workflow_dispatch: inputs: debug: From 9180bb752833e6178946378de108889c49fcd9dc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:37:12 +0100 Subject: [PATCH 014/459] added pylib and requirements.txt --- .github/workflows/python2.7.yaml | 6 ++++++ .github/workflows/python3.10.yaml | 6 ++++++ .github/workflows/python3.6.yaml | 6 ++++++ .github/workflows/python3.7.yaml | 6 ++++++ .github/workflows/python3.8.yaml | 6 ++++++ .github/workflows/python3.9.yaml | 6 ++++++ 6 files changed, 36 insertions(+) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml index 60b31b6bc..ff3e51dd1 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python2.7.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python2.7.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 939ae7337..de6f79440 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.10.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.10.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml index d6a61ea45..84dce85e0 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.6.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.6.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 70ca1f828..58ad6c210 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.7.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.7.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index fd6dcefb2..f1c806dc2 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.8.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.8.yaml workflow_dispatch: inputs: debug: diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index e8d331278..31c7d0ab8 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -21,12 +21,18 @@ on: - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.9.yaml pull_request: branches: - master - main paths: - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.9.yaml workflow_dispatch: inputs: debug: From 2fe17870b260fb815a2db3e63c0e7138b925f6b8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 015/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4bc47ad4e..b352785d0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4bc47ad4e3dc0d23f08054b6f4687a92869ec9f5 +Subproject commit b352785d0250a833a1b77f18f3efbd8d0a9038eb From 1f7dd4b40204da6a26f91bb930fbf3fb36674694 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 016/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index b22457e26..ffe46e95e 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit b22457e26204158f537524111be4e45669b82663 +Subproject commit ffe46e95e258b70b9e66f1e928291a273b93973b From 1d29f187221e74e7e05c9322a00402b8f9d865a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 017/459] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 300fd2a80..ad25bcb23 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 300fd2a80741eaf8dcf3e2900f507a5531355bd7 +Subproject commit ad25bcb23280f7307d8897111a238cc6b688f876 From 2ae16a579a126f1d43d69d9f4748a83bef4ec77a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:48:31 +0100 Subject: [PATCH 018/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 9a3f5c616..de060e1aa 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 9a3f5c616667cab76f98c13f2d8576610b877d1f +Subproject commit de060e1aa016757a2411b5ab8e8d35bf4750c9cb From 13675e4f89ec8b98b31c552cca802c7e3199c4c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 019/459] updated docker_pytools_alpine.yaml --- .github/workflows/docker_pytools_alpine.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index c3687b351..252e76f7b 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:alpine dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-alpine + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 3204f55973046b07e10b5148d85074e6a0498dbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 020/459] updated docker_pytools_centos.yaml --- .github/workflows/docker_pytools_centos.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml index 92b36e7d5..88474229f 100644 --- a/.github/workflows/docker_pytools_centos.yaml +++ b/.github/workflows/docker_pytools_centos.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:centos dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-centos + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 4fb936a61ebc471c3abb1bc378235ce3c343eeaf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 021/459] updated docker_pytools_debian.yaml --- .github/workflows/docker_pytools_debian.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index 8b34df315..876ccce1f 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:debian dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-debian + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From d4196934562d74e6a72599ec297fccd8003c1d10 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:31 +0100 Subject: [PATCH 022/459] updated docker_pytools_fedora.yaml --- .github/workflows/docker_pytools_fedora.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 190cf703b..0299270f3 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -31,6 +46,7 @@ jobs: ghcr.io/harisekhon/pytools:fedora dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-fedora + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 9e62e14fb9aa565b9dc08bb46e8f8814df9d4bfb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 18:54:32 +0100 Subject: [PATCH 023/459] updated docker_pytools_ubuntu.yaml --- .github/workflows/docker_pytools_ubuntu.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index fe5e51a95..535621a81 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -19,7 +19,22 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' + - .github/ + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false jobs: docker_build: @@ -33,6 +48,7 @@ jobs: ghcr.io/harisekhon/pytools:ubuntu dockerfile-repo: HariSekhon/Dockerfiles context: Dockerfiles/devops-python-tools-ubuntu + debug: ${{ github.event.inputs.debug }} secrets: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} From 898775ec901b4d4cb8ad6204c98c59a69fdd8d1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:01:47 +0100 Subject: [PATCH 024/459] added kics.yaml --- .github/workflows/kics.yaml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/kics.yaml diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml new file mode 100644 index 000000000..b73bedf6f --- /dev/null +++ b/.github/workflows/kics.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Kics + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kics: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Kics + uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master From 6216db9880e0ce935a1e264566dad52a911a1e3e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:05:30 +0100 Subject: [PATCH 025/459] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3aa702615..1561bd333 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Hari Sekhon - DevOps Python Tools [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) +[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) [![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) From f3cc7f678f6a75a4dba04b47ac38c59a5ae28d9a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:31:49 +0100 Subject: [PATCH 026/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b352785d0..df5a509bb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b352785d0250a833a1b77f18f3efbd8d0a9038eb +Subproject commit df5a509bb8c679914a137c8ff21597863fdd21ac From 131c4a984cb3848411c15b2036216c70ec5dbfbe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:31:49 +0100 Subject: [PATCH 027/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ffe46e95e..548ebc658 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ffe46e95e258b70b9e66f1e928291a273b93973b +Subproject commit 548ebc658c20f56600cac1bbd87ee1f267274696 From fbed6887015c0207021ef28ce82a4376260a009d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 28 Apr 2022 19:31:49 +0100 Subject: [PATCH 028/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index de060e1aa..b81a01bca 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit de060e1aa016757a2411b5ab8e8d35bf4750c9cb +Subproject commit b81a01bca4cca15dd9cf495bb253d1e27693cc7d From d3b9407383090a554a75ec40f452155c4895765b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 10 May 2022 12:01:24 +0100 Subject: [PATCH 029/459] updated URLs --- .appveyor.yml | 4 +- .buildkite/pipeline.yml | 4 +- .circleci/config.yml | 4 +- .cirrus.yml | 4 +- .concourse.yml | 6 +-- .drone.yml | 4 +- .editorconfig | 2 +- .github/ISSUE_TEMPLATE.md | 2 +- .github/workflows/docker_pytools_alpine.yaml | 2 +- .github/workflows/docker_pytools_centos.yaml | 2 +- .github/workflows/docker_pytools_debian.yaml | 2 +- .github/workflows/docker_pytools_fedora.yaml | 2 +- .github/workflows/docker_pytools_ubuntu.yaml | 2 +- .../ghcr_python_ubuntu.yaml.disabled | 2 +- .gitlab-ci.yml | 2 +- .gitmodules | 2 +- .gocd.yml | 6 +-- .semaphore/semaphore.yml | 4 +- .travis.yml | 4 +- .yamllint | 4 +- Jenkinsfile | 6 +-- Makefile | 10 ++--- README.md | 38 +++++++++---------- ambari_ams_metrics.sh | 4 +- ambari_blueprints.py | 4 +- ambari_cancel_all_requests.sh | 4 +- ambari_trigger_service_checks.py | 4 +- anonymize.py | 8 ++-- anonymize_custom.conf | 8 +++- anonymize_ignore.conf | 2 +- anonymize_parallel.sh | 4 +- aws_s3_presign.py | 4 +- aws_users_access_key_age.py | 8 ++-- aws_users_last_used.py | 8 ++-- aws_users_pw_last_used.py | 8 ++-- aws_users_unused_access_keys.py | 8 ++-- azure-pipelines.yml | 2 +- bitbucket-pipelines.yml | 4 +- buddy.yml | 6 +-- buildspec.yml | 2 +- center.py | 4 +- cloudbuild.yaml | 2 +- cloudera_navigator_tables_used.py | 4 +- cloudera_navigator_tables_used_postgres.py | 4 +- codefresh.yml | 4 +- codeship.yml | 2 +- crunch_accounting_csv_statement_converter.py | 4 +- ...unch_accounting_csv_statement_converter.sh | 2 +- ...unch_accounting_csv_statement_converter.sh | 2 +- .../lib.sh | 2 +- docker_pull_all_images.sh | 4 +- docker_pull_all_images_all_tags.sh | 4 +- docker_pull_all_tags.sh | 4 +- docker_registry_show_tags.py | 4 +- dockerfiles_check_git_branches.py | 4 +- dockerfiles_check_git_tags.py | 4 +- dockerhub_search.py | 6 +-- dockerhub_show_tags.py | 4 +- find_active_apache_drill.py | 4 +- find_active_apache_drill2.py | 4 +- find_active_cassandra.py | 4 +- find_active_elasticsearch.py | 4 +- find_active_hadoop2_namenode.py | 4 +- find_active_hadoop_namenode.py | 4 +- find_active_hadoop_yarn_resource_manager.py | 4 +- find_active_hbase_master.py | 4 +- find_active_hbase_stargate.py | 4 +- find_active_hbase_thrift.py | 4 +- find_active_impala.py | 4 +- find_active_impala_catalog.py | 4 +- find_active_impala_statestore.py | 4 +- find_active_kubernetes_api.py | 4 +- find_active_oozie.py | 4 +- find_active_presto_coordinator.py | 4 +- find_active_server.py | 6 +-- find_active_solrcloud.py | 4 +- find_duplicate_files.py | 4 +- find_missing_files_in_sequence.py | 2 +- getent.py | 4 +- git_check_branches_upstream.py | 6 +-- hbase_compact_tables.py | 4 +- hbase_flush_tables.py | 4 +- hbase_generate_data.py | 4 +- hbase_region_requests.py | 4 +- hbase_regions_by_memstore_size.py | 4 +- hbase_regions_by_size.py | 4 +- hbase_regions_least_used.py | 4 +- hbase_regionserver_requests.py | 4 +- hbase_scan_table_column_names.sh | 4 +- hbase_show_table_region_ranges.py | 4 +- hbase_table_region_row_distribution.py | 4 +- hbase_table_regions_by_regionserver.sh | 4 +- hbase_table_regionserver_distribution.sh | 4 +- hbase_table_row_key_distribution.py | 4 +- hbase_uncompacted_regions.sh | 4 +- hdfs_files_native_checksums.jy | 4 +- hdfs_files_stats.jy | 4 +- hdfs_find_replication_factor_1.py | 4 +- hdfs_time_block_reads.jy | 4 +- headtail.py | 4 +- hexanonymize.py | 2 +- hive_compute_table_stats.py | 4 +- hive_foreach_table.py | 4 +- hive_schemas_csv.py | 4 +- hive_tables_column_counts.py | 4 +- hive_tables_list.py | 4 +- hive_tables_locations.py | 4 +- hive_tables_metadata.py | 4 +- hive_tables_null_columns.py | 4 +- hive_tables_null_rows.py | 4 +- hive_tables_row_column_counts.py | 4 +- hive_tables_row_counts.py | 4 +- hive_tables_row_counts_any_nulls.py | 4 +- impala_compute_table_stats.py | 4 +- impala_foreach_table.py | 4 +- impala_schemas_csv.py | 4 +- impala_tables_column_counts.py | 4 +- impala_tables_list.py | 4 +- impala_tables_locations.py | 4 +- impala_tables_metadata.py | 4 +- impala_tables_null_columns.py | 4 +- impala_tables_null_rows.py | 4 +- impala_tables_row_column_counts.py | 4 +- impala_tables_row_counts.py | 4 +- impala_tables_row_counts_any_nulls.py | 4 +- ipython_notebook_pyspark.py | 4 +- json_docs_to_bulk_multiline.py | 4 +- json_to_xml.py | 2 +- json_to_yaml.py | 2 +- jython_autoinstall.exp | 4 +- jython_install.sh | 4 +- lib/hive_impala_cli.py | 4 +- lib/postgres_cli.py | 4 +- opentsdb_import_metric_distribution.py | 4 +- opentsdb_list_metrics.sh | 4 +- opentsdb_list_metrics_hbase.sh | 4 +- pig_udfs.jy | 2 +- python_find_library_path.py | 4 +- pythonpath.py | 4 +- quay_show_tags.py | 4 +- serf_event_handler.py | 4 +- setup/apk-packages-dev.txt | 4 +- setup/apk-packages-pip.txt | 4 +- setup/apk-packages.txt | 4 +- setup/bootstrap.sh | 6 +-- setup/brew-packages.txt | 4 +- setup/ci_bootstrap.sh | 4 +- setup/deb-packages-dev.txt | 4 +- setup/deb-packages-optional.txt | 4 +- setup/deb-packages-pip.txt | 4 +- setup/deb-packages.txt | 4 +- setup/gocd_config_repo.json | 2 +- setup/jenkins-job.xml | 4 +- setup/rpm-packages-dev.txt | 4 +- setup/rpm-packages-optional.txt | 4 +- setup/rpm-packages-pip.txt | 4 +- setup/rpm-packages.txt | 4 +- shippable.yml | 4 +- sonar-project.properties | 10 ++--- spark_avro_to_parquet.py | 4 +- spark_csv_to_avro.py | 4 +- spark_csv_to_parquet.py | 4 +- spark_json_to_avro.py | 4 +- spark_json_to_parquet.py | 4 +- spark_parquet_to_avro.py | 4 +- strip_ansi_escape_codes.py | 4 +- tests/all.sh | 4 +- tests/check.sh | 4 +- tests/compile.sh | 4 +- tests/data/add_ou.ldif | 4 +- tests/data/ldap_download.sh | 2 +- tests/data/ldap_upload.sh | 2 +- tests/docker/apache-drill-docker-compose.yml | 4 +- tests/docker/common.yml | 4 +- tests/docker/elasticsearch-docker-compose.yml | 4 +- ...lasticsearch-elastic.co-docker-compose.yml | 4 +- tests/docker/hadoop-docker-compose.yml | 4 +- tests/docker/hbase-docker-compose.yml | 4 +- tests/docker/presto-dev-docker-compose.yml | 4 +- tests/docker/presto-docker-compose.yml | 4 +- tests/docker/registry-docker-compose.yml | 4 +- tests/docker/solrcloud-docker-compose.yml | 4 +- tests/excluded.sh | 4 +- tests/help.sh | 4 +- tests/python3.sh | 4 +- tests/syntax.sh | 4 +- tests/test_anonymize.sh | 4 +- tests/test_apache-drill.sh | 4 +- tests/test_center.sh | 2 +- tests/test_docker.sh | 4 +- tests/test_docker_registry_show_tags.sh | 4 +- tests/test_dockerfiles_check_git_branches.sh | 6 +-- tests/test_dockerfiles_check_git_tags.sh | 6 +-- tests/test_dockerhub_search.sh | 4 +- tests/test_dockerhub_show_tags.sh | 4 +- tests/test_elasticsearch.sh | 4 +- tests/test_find_active_server.sh | 2 +- tests/test_find_duplicate_files.sh | 4 +- tests/test_find_python_library_path.sh | 4 +- tests/test_getent.sh | 4 +- tests/test_git_check_branches_upstream.sh | 6 +-- tests/test_hadoop.sh | 4 +- tests/test_hbase.sh | 4 +- tests/test_headtail.sh | 4 +- tests/test_json.sh | 4 +- tests/test_json_docs_to_bulk_multiline.sh | 4 +- tests/test_json_to_xml.sh | 4 +- tests/test_json_to_yaml.sh | 4 +- tests/test_opentsdb.sh | 4 +- tests/test_presto.sh | 4 +- tests/test_quay_show_tags.sh | 4 +- tests/test_serf_event_handler.sh | 4 +- tests/test_solrcloud.sh | 4 +- tests/test_spark_csv_to_avro.sh | 4 +- tests/test_spark_csv_to_parquet.sh | 4 +- tests/test_spark_json_to_avro.sh | 4 +- tests/test_spark_json_to_parquet.sh | 4 +- tests/test_spark_z_avro_to_parquet.sh | 4 +- tests/test_spark_z_parquet_to_avro.sh | 4 +- tests/test_timeout.sh | 2 +- tests/test_validate_avro.sh | 4 +- tests/test_validate_cson.sh | 4 +- tests/test_validate_csv.sh | 4 +- tests/test_validate_ini.sh | 4 +- tests/test_validate_json.sh | 4 +- tests/test_validate_ldap_ldif.sh | 4 +- tests/test_validate_multimedia.sh | 4 +- tests/test_validate_parquet.sh | 4 +- tests/test_validate_toml.sh | 4 +- tests/test_validate_xml.sh | 4 +- tests/test_validate_yaml.sh | 4 +- tests/test_welcome.sh | 4 +- tests/test_xml_to_json.sh | 4 +- tests/test_xml_to_yaml.sh | 4 +- tests/test_yamllint.sh | 4 +- tests/utils.sh | 4 +- timeout.py | 4 +- travis_debug_session.py | 4 +- travis_last_log.py | 4 +- urldecode.py | 4 +- urlencode.py | 4 +- validate_avro.py | 4 +- validate_cson.py | 4 +- validate_csv.py | 4 +- validate_ini.py | 4 +- validate_ini2.py | 4 +- validate_json.py | 4 +- validate_ldap_ldif.py | 4 +- validate_multimedia.py | 4 +- validate_parquet.py | 4 +- validate_toml.py | 4 +- validate_xml.py | 4 +- validate_yaml.py | 4 +- welcome.py | 4 +- wercker.yml | 4 +- xml_to_json.py | 2 +- xml_to_yaml.py | 2 +- yaml_to_json.py | 2 +- 258 files changed, 532 insertions(+), 528 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 1551a3473..b706d6f72 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://www.appveyor.com/docs/appveyor-yml/ diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 53dd7405b..93f047c02 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # BuildKite Pipeline diff --git a/.circleci/config.yml b/.circleci/config.yml index 37332c73c..5dbde36b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,13 +5,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Master Template with more advanced config: diff --git a/.cirrus.yml b/.cirrus.yml index 7f1425a4c..d544aeb85 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://cirrus-ci.org/guide/writing-tasks/ diff --git a/.concourse.yml b/.concourse.yml index 9ed7da3e2..42b806f26 100644 --- a/.concourse.yml +++ b/.concourse.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # resources: @@ -18,7 +18,7 @@ resources: icon: github-circle type: git source: - uri: https://github.com/harisekhon/devops-python-tools + uri: https://github.com/HariSekhon/DevOps-Python-tools branch: master #- name: daily # type: time diff --git a/.drone.yml b/.drone.yml index 790bdcf4c..4d36c9db2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6,13 +6,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://docs.drone.io/quickstart/cli/ diff --git a/.editorconfig b/.editorconfig index 268166e72..aac36b310 100644 --- a/.editorconfig +++ b/.editorconfig @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 5f04c0a58..aa46b9847 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,3 +1,3 @@ Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell. -You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively. +You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) respectively. diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index 252e76f7b..8ce10e111 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/docker_pytools_centos.yaml index 88474229f..e8db3ff9c 100644 --- a/.github/workflows/docker_pytools_centos.yaml +++ b/.github/workflows/docker_pytools_centos.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index 876ccce1f..2b92c8282 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 0299270f3..92a93a6b4 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index 535621a81..d6a4ddcb0 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.github/workflows/ghcr_python_ubuntu.yaml.disabled b/.github/workflows/ghcr_python_ubuntu.yaml.disabled index fdd5ad61b..c4503bc13 100644 --- a/.github/workflows/ghcr_python_ubuntu.yaml.disabled +++ b/.github/workflows/ghcr_python_ubuntu.yaml.disabled @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af2a6a22b..f9a691721 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: Sun Feb 23 19:02:10 2020 +0000 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # diff --git a/.gitmodules b/.gitmodules index 2bb07ff73..455bce3c9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "pylib"] path = pylib - url = https://github.com/harisekhon/pylib + url = https://github.com/HariSekhon/pylib branch = master [submodule "bash-tools"] path = bash-tools diff --git a/.gocd.yml b/.gocd.yml index 69d289bf4..d3a89a0ef 100644 --- a/.gocd.yml +++ b/.gocd.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://github.com/tomzo/gocd-yaml-config-plugin#setup @@ -26,7 +26,7 @@ pipelines: display_order: -1 materials: git: - git: https://github.com/harisekhon/devops-python-tools + git: https://github.com/HariSekhon/DevOps-Python-tools shallow_clone: false auto_update: true branch: master diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 99040a203..dd256f26c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -4,13 +4,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ diff --git a/.travis.yml b/.travis.yml index 71348b0bc..47f2f80b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,14 @@ # Author: Hari Sekhon # Date: 2014-11-29 01:02:47 +0000 (Sat, 29 Nov 2014) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/.yamllint b/.yamllint index b5e6f8448..e29153b7f 100644 --- a/.yamllint +++ b/.yamllint @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/Jenkinsfile b/Jenkinsfile index 7739f92e3..c4ff34fb2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,13 +4,13 @@ // Author: Hari Sekhon // Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017) // -// https://github.com/harisekhon/devops-python-tools +// https://github.com/HariSekhon/DevOps-Python-tools // // License: see accompanying Hari Sekhon LICENSE file // // If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish // -// https://www.linkedin.com/in/harisekhon +// https://www.linkedin.com/in/HariSekhon // // ========================================================================== // @@ -49,7 +49,7 @@ pipeline { stages { stage ('Checkout') { steps { - checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]]) + checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/HariSekhon/DevOps-Python-tools']]]) } } diff --git a/Makefile b/Makefile index b7f961b67..ae61cbc5e 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules to non-system python otherwise they're not found by programs. @@ -21,15 +21,15 @@ # # Alpine: # -# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make +# apk add --no-cache git make && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # # Debian / Ubuntu: # -# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make +# apt-get update && apt-get install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # # RHEL / CentOS: # -# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make +# yum install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # =================== diff --git a/README.md b/README.md index 1561bd333..535e5389a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ Hari Sekhon - DevOps Python Tools ================================= -[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/stargazers) -[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/harisekhon/devops-python-tools/network) +[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) @@ -116,10 +116,10 @@ DevOps, Cloud, Big Data, NoSQL, Python & Linux tools. All programs have `--help` See Also: - - [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - - [DevOps Perl Tools](https://github.com/harisekhon/devops-perl-tools) + - [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) + - [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) - [DevOps Golang Tools](https://github.com/HariSekhon/DevOps-Golang-tools) - - [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) + - [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) repos which contains hundreds more scripts and programs for Cloud, Big Data, SQL, NoSQL, Web and Linux. @@ -127,7 +127,7 @@ Hari Sekhon Cloud & Big Data Contractor, United Kingdom -[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/harisekhon/) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/HariSekhon/) ###### (you're welcome to connect with me on LinkedIn) ##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### @@ -156,7 +156,7 @@ curl -L https://git.io/python-bootstrap | sh or manually: ``` -git clone https://github.com/harisekhon/devops-python-tools pytools +git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools make ``` @@ -166,9 +166,9 @@ To only install pip dependencies for a single script, you can just type make and make anonymize.pyc ``` -Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/devops-python-tools#detailed-build-instructions) further down for more information. +Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/DevOps-Python-tools#detailed-build-instructions) further down for more information. -Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/harisekhon/devops-python-tools#jython-for-hadoop-utils) for details. +Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/HariSekhon/DevOps-Python-tools#jython-for-hadoop-utils) for details. ### Usage @@ -290,7 +290,7 @@ Environment variables are supported for convenience and also to hide credentials - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) + - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI @@ -321,7 +321,7 @@ The automated build will use 'sudo' to install required Python PyPI libraries to Enter the pytools directory and run git submodule init and git submodule update to fetch my library repo: ``` -git clone https://github.com/harisekhon/devops-python-tools pytools +git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools git submodule init git submodule update @@ -333,7 +333,7 @@ sudo pip install -r requirements.txt Download the DevOps Python Tools and Pylib git repos as zip files: -https://github.com/HariSekhon/devops-python-tools/archive/master.zip +https://github.com/HariSekhon/DevOps-Python-tools/archive/master.zip https://github.com/HariSekhon/pylib/archive/master.zip @@ -433,7 +433,7 @@ Then add the Jython install bin directory to the $PATH or specify the full path #### Configuration for Strict Domain / FQDN validation #### -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/harisekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). #### Python SSL certificate verification problems @@ -457,9 +457,9 @@ If you update often and want to just quickly git pull + submodule update but ski ### Testing [Continuous Integration](https://travis-ci.org/HariSekhon/devops-python-tools) is run on this repo with tests for success and failure scenarios: -- unit tests for the custom supporting [python library](https://github.com/harisekhon/pylib) +- unit tests for the custom supporting [python library](https://github.com/HariSekhon/pylib) - integration tests of the top level programs using the libraries for things like option parsing -- [functional tests](https://github.com/HariSekhon/devops-python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/) +- [functional tests](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/) To trigger all tests run: @@ -475,7 +475,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ### See Also ### -- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 700+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 700+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery @@ -483,7 +483,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments -- [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +- [The Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) - [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... @@ -491,9 +491,9 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 -- [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule +- [PyLib](https://github.com/HariSekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule -- [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library +- [Perl Lib](https://github.com/HariSekhon/lib) - Perl version of above library From bccc01b7473cf51d3a5be02a09f379fe3ff569fe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:42 +0100 Subject: [PATCH 167/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 0caddf7a5..5e7dfbdc8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 0caddf7a58297dd75d19ecfea71eb2206853d4c4 +Subproject commit 5e7dfbdc8cea422b6856153eea3c01df4c84a60c From 5a0a33122adaa05720ea69222186feae21887c6f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:42 +0100 Subject: [PATCH 168/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 357fcd97c..4515bb958 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 357fcd97c3ccf3b6f89aa07ef71c994504e7c7c8 +Subproject commit 4515bb958f4b27c7c5854ba77ee665b41852dd5b From cf3544f48c0168b9be909b530a3c071d9b46aad7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:43 +0100 Subject: [PATCH 169/459] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ff707a116..d4eedc0e2 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ff707a1168889fb3ad759c030f3fc633d23d800f +Subproject commit d4eedc0e2c68c9a43838c89a1d09abf10927a566 From 194ca3daa90416a88ee399a7b9e7c3f0d7bd90c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:04:43 +0100 Subject: [PATCH 170/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 7373f570f..8d838daa6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 7373f570ffb1a790ec576ef4c6cf52dcf0785ba0 +Subproject commit 8d838daa669dc05d497d5fb2fa1aef631c613acf From 2b13a79338d74de5b5ea78f59cd02f79484e1c61 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:28:26 +0100 Subject: [PATCH 171/459] renamed debian_8.yaml to debian_8.yaml.disabled --- .github/workflows/{debian_8.yaml => debian_8.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{debian_8.yaml => debian_8.yaml.disabled} (100%) diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml.disabled similarity index 100% rename from .github/workflows/debian_8.yaml rename to .github/workflows/debian_8.yaml.disabled From 8ef52b84f1c5e3aa90439b660ec903ee8c132e05 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:37:40 +0100 Subject: [PATCH 172/459] added debian_11.yaml --- .github/workflows/debian_11.yaml | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/debian_11.yaml diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml new file mode 100644 index 000000000..de3b4b96f --- /dev/null +++ b/.github/workflows/debian_11.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 11 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_11.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:11 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} From ed38e3efe36f2d5fc3eae3f90b6f50d82d538a47 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 22:49:50 +0100 Subject: [PATCH 173/459] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 152556544..af9216972 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ [![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) -[![Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+8%22) [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) +[![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) [![CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22) [![CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22) [![CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22) From 5ba4480dc89627d3c39d1f8de03f3450d709fa7c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:42:16 +0100 Subject: [PATCH 174/459] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index af9216972..c6ae3495f 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,9 @@ [![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) [![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) [![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) -[![Wercker](https://app.wercker.com/status/b40735fb89e7d989dbaf5659a9af9a20/s/master "wercker status")](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs) [![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) [![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml) [![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) - [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) From 824e98dd0d38a3382c69dac5eb7e85b1056836ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:49:30 +0100 Subject: [PATCH 175/459] renamed python3.6.yaml to python3.6.yaml.disabled --- .github/workflows/{python3.6.yaml => python3.6.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python3.6.yaml => python3.6.yaml.disabled} (100%) diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml.disabled similarity index 100% rename from .github/workflows/python3.6.yaml rename to .github/workflows/python3.6.yaml.disabled From 7408f3ce0898dea20692929e949e336edd98dfe8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:49:36 +0100 Subject: [PATCH 176/459] renamed python2.7.yaml to python2.7.yaml.disabled --- .github/workflows/{python2.7.yaml => python2.7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python2.7.yaml => python2.7.yaml.disabled} (100%) diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml.disabled similarity index 100% rename from .github/workflows/python2.7.yaml rename to .github/workflows/python2.7.yaml.disabled From 83b8b0506a21b3636ba4f75627f965c01fb0545f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:51:59 +0100 Subject: [PATCH 177/459] added python3.11.yaml --- .github/workflows/python3.11.yaml | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/python3.11.yaml diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml new file mode 100644 index 000000000..8e2d041ac --- /dev/null +++ b/.github/workflows/python3.11.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.11 + +on: + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.11 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: "3.11" + caches: apt pip + debug: ${{ github.event.inputs.debug }} From 1cfb67773c00608d7fe6fc109f46a968747f63bc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:53:20 +0100 Subject: [PATCH 178/459] updated README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index c6ae3495f..5b9392d62 100644 --- a/README.md +++ b/README.md @@ -101,12 +101,11 @@ [![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) [![Python versions](https://img.shields.io/badge/Python-2.7+-3776AB?logo=python&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools) -[![Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+2.7%22) -[![Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.6%22) [![Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22) [![Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22) [![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) +[![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) [![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) [![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) From d824e50952a1bb22fef5061544e37305f6b37623 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:56:31 +0100 Subject: [PATCH 179/459] renamed pypy2.yaml to pypy2.yaml.disabled --- .github/workflows/{pypy2.yaml => pypy2.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pypy2.yaml => pypy2.yaml.disabled} (100%) diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml.disabled similarity index 100% rename from .github/workflows/pypy2.yaml rename to .github/workflows/pypy2.yaml.disabled From 9091c212bd734a1eb605f29d3ecf516fcb12cbcb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:56:31 +0100 Subject: [PATCH 180/459] renamed pypy3.yaml to pypy3.yaml.disabled --- .github/workflows/{pypy3.yaml => pypy3.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pypy3.yaml => pypy3.yaml.disabled} (100%) diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml.disabled similarity index 100% rename from .github/workflows/pypy3.yaml rename to .github/workflows/pypy3.yaml.disabled From 8bfd91c7b4649060b5704ac3f092d62ae667ca11 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 181/459] renamed ubuntu_14.04.yaml to ubuntu_14.04.yaml.disabled --- .../workflows/{ubuntu_14.04.yaml => ubuntu_14.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_14.04.yaml => ubuntu_14.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_14.04.yaml rename to .github/workflows/ubuntu_14.04.yaml.disabled From a4139b161723a350733d1ce22aab019e536a4c80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 182/459] renamed ubuntu_16.04.yaml to ubuntu_16.04.yaml.disabled --- .../workflows/{ubuntu_16.04.yaml => ubuntu_16.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_16.04.yaml => ubuntu_16.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_16.04.yaml rename to .github/workflows/ubuntu_16.04.yaml.disabled From 46ab2bf3cf7fbb872f451c9d0590a9b2dc8ce803 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 183/459] renamed ubuntu_18.04.yaml to ubuntu_18.04.yaml.disabled --- .../workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml rename to .github/workflows/ubuntu_18.04.yaml.disabled From d3e2d0be996674fe5ccedc47a9bce75b436532fb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 184/459] renamed ubuntu_20.04.yaml to ubuntu_20.04.yaml.disabled --- .../workflows/{ubuntu_20.04.yaml => ubuntu_20.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_20.04.yaml => ubuntu_20.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_20.04.yaml rename to .github/workflows/ubuntu_20.04.yaml.disabled From 1f744ad1dbf1d66fddae98a6244c7f1f35ece41b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 18 Apr 2023 23:58:28 +0100 Subject: [PATCH 185/459] renamed ubuntu_github.yaml to ubuntu_github.yaml.disabled --- .../workflows/{ubuntu_github.yaml => ubuntu_github.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_github.yaml => ubuntu_github.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_github.yaml rename to .github/workflows/ubuntu_github.yaml.disabled From 72c62d30af56d108aa1554fb4884dd6abe635ba0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:02:48 +0100 Subject: [PATCH 186/459] renamed ubuntu_18.04.yaml.disabled to ubuntu_18.04.yaml --- .../workflows/{ubuntu_18.04.yaml.disabled => ubuntu_18.04.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml.disabled => ubuntu_18.04.yaml} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml.disabled rename to .github/workflows/ubuntu_18.04.yaml From 62d24176064c6b4dac09d71938d5f0d6e2b34c58 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:03:08 +0100 Subject: [PATCH 187/459] renamed ubuntu_20.04.yaml.disabled to ubuntu_20.04.yaml --- .../workflows/{ubuntu_20.04.yaml.disabled => ubuntu_20.04.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_20.04.yaml.disabled => ubuntu_20.04.yaml} (100%) diff --git a/.github/workflows/ubuntu_20.04.yaml.disabled b/.github/workflows/ubuntu_20.04.yaml similarity index 100% rename from .github/workflows/ubuntu_20.04.yaml.disabled rename to .github/workflows/ubuntu_20.04.yaml From 209f3bfd5641f436862ffca3dece34d0bf81b3fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:05:48 +0100 Subject: [PATCH 188/459] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5b9392d62..32ee3ce53 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,6 @@ [![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) -[![Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+14.04%22) -[![Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+16.04%22) [![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) From e550318685f87e7e2c25b400e634175915a8debe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:06:12 +0100 Subject: [PATCH 189/459] renamed ubuntu_github.yaml.disabled to ubuntu_github.yaml --- .../workflows/{ubuntu_github.yaml.disabled => ubuntu_github.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_github.yaml.disabled => ubuntu_github.yaml} (100%) diff --git a/.github/workflows/ubuntu_github.yaml.disabled b/.github/workflows/ubuntu_github.yaml similarity index 100% rename from .github/workflows/ubuntu_github.yaml.disabled rename to .github/workflows/ubuntu_github.yaml From c0e1bb251c0c9028b9f5b0907788e7766f346acf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:07:38 +0100 Subject: [PATCH 190/459] renamed ubuntu_18.04.yaml to ubuntu_18.04.yaml.disabled --- .../workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_18.04.yaml => ubuntu_18.04.yaml.disabled} (100%) diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml.disabled similarity index 100% rename from .github/workflows/ubuntu_18.04.yaml rename to .github/workflows/ubuntu_18.04.yaml.disabled From cbf21a97aabc95628d71f773f08229cab9ac8614 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:10:29 +0100 Subject: [PATCH 191/459] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32ee3ce53..138775975 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,8 @@ [![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) -[![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) +[![Ubuntu 22.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2022.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+22.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) From a5ad66e4090134ecc51d4dbb869251e63f8f27c1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:40 +0100 Subject: [PATCH 192/459] renamed centos.yaml to centos.yaml.disabled --- .github/workflows/{centos.yaml => centos.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos.yaml => centos.yaml.disabled} (100%) diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml.disabled similarity index 100% rename from .github/workflows/centos.yaml rename to .github/workflows/centos.yaml.disabled From af02db5511f56a40b40a90702619c5dfd8f600a3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:40 +0100 Subject: [PATCH 193/459] renamed centos7.yaml to centos7.yaml.disabled --- .github/workflows/{centos7.yaml => centos7.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos7.yaml => centos7.yaml.disabled} (100%) diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml.disabled similarity index 100% rename from .github/workflows/centos7.yaml rename to .github/workflows/centos7.yaml.disabled From 5b91ee48587426db040ac1afb31fbfef8f1fbeff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:26:41 +0100 Subject: [PATCH 194/459] renamed centos8.yaml to centos8.yaml.disabled --- .github/workflows/{centos8.yaml => centos8.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{centos8.yaml => centos8.yaml.disabled} (100%) diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml.disabled similarity index 100% rename from .github/workflows/centos8.yaml rename to .github/workflows/centos8.yaml.disabled From 79fa99d693ea366638e1ee6c38bd5dec65e4ee43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:28:28 +0100 Subject: [PATCH 195/459] updated README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 138775975..72b1c3aab 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,6 @@ [![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) [![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) -[![CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22) -[![CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22) -[![CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22) [![Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Fedora%22) [![Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine%22) [![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) From 78ba04ce9d93aedfee00e09cb99c083884a53e1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:48:44 +0100 Subject: [PATCH 196/459] updated serf_event_handler.py --- serf_event_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serf_event_handler.py b/serf_event_handler.py index 455d7bee1..c0d5cacbb 100755 --- a/serf_event_handler.py +++ b/serf_event_handler.py @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.2' +__version__ = '0.2.3' class SerfEventHandler(CLI): @@ -95,7 +95,7 @@ def add_options(self): def enable_commands(self): if self.event in ['query', 'event']: - cmd = None + cmd = '' if self.event == 'query': cmd = self.query_name elif self.event == 'user': From f66248cc4545fe236327911857290247eba1a701 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:49:41 +0100 Subject: [PATCH 197/459] updated spark_json_to_avro.py --- spark_json_to_avro.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index 2922c7f7f..d53a4bffa 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -109,7 +109,6 @@ def run(self): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): df = sqlContext.read.json(json_file) else: From 8446d976d4efcee77aad458c5086d16c86a74255 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 00:53:59 +0100 Subject: [PATCH 198/459] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 72b1c3aab..e12942015 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools) [![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) From d80f92c71f9aefed3a161dc9cf719766ad4c1eef Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:05:15 +0100 Subject: [PATCH 199/459] renamed docker_pytools_centos.yaml to *_centos.yaml.disabled --- .../{docker_pytools_centos.yaml => *_centos.yaml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{docker_pytools_centos.yaml => *_centos.yaml.disabled} (100%) diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/*_centos.yaml.disabled similarity index 100% rename from .github/workflows/docker_pytools_centos.yaml rename to .github/workflows/*_centos.yaml.disabled From 930a2c341a40292160d34cfc827a6b8358f306a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:06:10 +0100 Subject: [PATCH 200/459] updated README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index e12942015..c301b354d 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,6 @@ [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) -[![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) [![Docker Build (Fedora)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml) [![Docker Build (Ubuntu)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml) From 2f629017ce6a08b0db62bff40c1a90f5f801ab53 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:11:11 +0100 Subject: [PATCH 201/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 5e7dfbdc8..bccee5cd7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 5e7dfbdc8cea422b6856153eea3c01df4c84a60c +Subproject commit bccee5cd73f9bb86ab84609681defcea0ec23b9d From a109075a96e1d21f3dbe6cc21dd76b4a0706359e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:11:11 +0100 Subject: [PATCH 202/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 4515bb958..47048ee0c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 4515bb958f4b27c7c5854ba77ee665b41852dd5b +Subproject commit 47048ee0c39d1e88a10d09fec64333b2de8e906b From 5bdc0c0db940cbc74019b7b37508c6943e0b65f4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:33:03 +0100 Subject: [PATCH 203/459] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index c301b354d..edb49e6b1 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,6 @@ [![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) [![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) -[![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) -[![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) [git.io/pytools](https://git.io/pytools) From e2f37bf682f28263a993209c5e52bcef272025d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:36:37 +0100 Subject: [PATCH 204/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index bccee5cd7..995ad2de8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit bccee5cd73f9bb86ab84609681defcea0ec23b9d +Subproject commit 995ad2de8486a96487e68c859c4f353a381b91d7 From 8bb7bc784d8f0ad98dbefd691ee8bc460f727330 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 01:36:37 +0100 Subject: [PATCH 205/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 47048ee0c..7c0ba3afe 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 47048ee0c39d1e88a10d09fec64333b2de8e906b +Subproject commit 7c0ba3afe47f47af186cadab75b508a69d4411fe From 383114f4538c6ccc9f2443d5c53d96e6f2c3d526 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:39:36 +0100 Subject: [PATCH 206/459] added ubuntu_22.04.yaml --- .github/workflows/ubuntu_22.04.yaml | 85 +++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/ubuntu_22.04.yaml diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml new file mode 100644 index 000000000..b9bf17a95 --- /dev/null +++ b/.github/workflows/ubuntu_22.04.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 22.04 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_22.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:22.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} From 2769fba289020e9b94311b1f1bee3bcefc02f60e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:43:07 +0100 Subject: [PATCH 207/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 995ad2de8..fe25b15b0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 995ad2de8486a96487e68c859c4f353a381b91d7 +Subproject commit fe25b15b096e5e667b0451ebc9f68cc86c87384f From 6bfe3eb1a41357c469d3895b6739dfea28d7fab3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 14:43:07 +0100 Subject: [PATCH 208/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 7c0ba3afe..0a225da5c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 7c0ba3afe47f47af186cadab75b508a69d4411fe +Subproject commit 0a225da5c8aa09653d53dd1aaad545d200ec7e7a From 55827d1746db9186a49880e91d4f6d711058e7fc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 18:50:55 +0100 Subject: [PATCH 209/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index fe25b15b0..2295778d8 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit fe25b15b096e5e667b0451ebc9f68cc86c87384f +Subproject commit 2295778d8039333abc3e159754641b348cf79d94 From 5491f93ea4295dd3a87c7541a5182b146baadfe7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 18:50:56 +0100 Subject: [PATCH 210/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0a225da5c..6e5e84d30 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0a225da5c8aa09653d53dd1aaad545d200ec7e7a +Subproject commit 6e5e84d301329d0386b8a3468e4f7b34700d3e32 From a9c81105474dbfad815563507e21fa7b6f5d8785 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 19:07:39 +0100 Subject: [PATCH 211/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 2295778d8..37dede02a 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 2295778d8039333abc3e159754641b348cf79d94 +Subproject commit 37dede02a5a184087798740bfbb36b90cf5ed685 From 3680e4c7ab65c6ef873bdf8151fe86efe68fd16a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 19 Apr 2023 19:07:39 +0100 Subject: [PATCH 212/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6e5e84d30..5f2f181a9 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6e5e84d301329d0386b8a3468e4f7b34700d3e32 +Subproject commit 5f2f181a96f884acd0340125aab776414c343b68 From 8865ba8731144538dd0baba23d1ccb787c5988a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 May 2023 17:50:27 +0100 Subject: [PATCH 213/459] added --unspace functionality --- center.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/center.py b/center.py index 4856ea0f7..b9cbac5f1 100755 --- a/center.py +++ b/center.py @@ -44,7 +44,8 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.2' +__version__ = '0.5.0' + class Center(CLI): @@ -56,7 +57,9 @@ def __init__(self): # this doesn't put enough spaces around ampersands, eg. in "Auth & Config" #self.re_bound = re.compile(r'(\b)') self.re_spaces = re.compile(r'(\s)') + self.re_multiple_spaces = re.compile(r'(\s){2}') self.re_chars = re.compile(r'([^\s])(?!\s)') + self.re_chars_spaced = re.compile(r'([^\s])\s') self.timeout_default = None def add_options(self): @@ -66,11 +69,16 @@ def add_options(self): help='No comment prefix handling') self.add_opt('-s', '--space', action='store_true', default=False, help='Space all chars out, makes bigger headings') + self.add_opt('-u', '--unspace', action='store_true', default=False, + help='Removes spaces betweeen chars out, the inverse of --space') def run(self): log_option('width', self.get_opt('width')) log_option('no comment prefix', self.get_opt('no_comment')) log_option('space chars', self.get_opt('space')) + log_option('unspace chars', self.get_opt('unspace')) + if self.get_opt('space') and self.get_opt('unspace'): + self.usage("--space and --unspace are mutually exclusive!") if self.args: self.process_line(' '.join(self.args)) else: @@ -83,6 +91,11 @@ def space(self, line): line = self.re_chars.sub(r'\1 ', line) return line + def unspace(self, line): + line = self.re_chars_spaced.sub(r'\1', line) + line = self.re_multiple_spaces.sub(r'\1', line) + return line + def process_line(self, line): char = '' if not line: @@ -102,9 +115,12 @@ def process_line(self, line): line = line.lstrip(char) if self.get_opt('space'): line = self.space(line) + if self.get_opt('unspace'): + line = self.unspace(line) line = line.strip() side = int(max((self.get_opt('width') - len(line)) / 2, 0)) print(char + ' ' * side + line) + if __name__ == '__main__': Center().main() From ad1c2e05e9ebe50dcc9829209328c2fa9c3b9016 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 1 May 2023 17:50:49 +0100 Subject: [PATCH 214/459] updated .pylintrc --- .pylintrc | 751 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 508 insertions(+), 243 deletions(-) diff --git a/.pylintrc b/.pylintrc index c20ac8f94..1c56d5de1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,386 +1,651 @@ -[MASTER] +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y L i n t C o n f i g +# ============================================================================ # + +# pylint --generate-rcfile >> .pylintrc + +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= -# Specify a configuration file. -#rcfile= +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 -# Pickle collected data for later comparisons. -persistent=yes +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 -# List of plugins (as comma separated values of python modules names) to load, +# List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= -# Use multiple processes to speed up Pylint. -jobs=1 +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. -optimize-ast=no +[BASIC] -[MESSAGES CONTROL] +# Naming style matching correct argument names. +argument-naming-style=snake_case -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= +# Naming style matching correct attribute names. +attr-naming-style=snake_case -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111,consider-using-f-string +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata -[REPORTS] +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text +# Naming style matching correct class attribute names. +class-attribute-naming-style=any -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= -# Tells whether to display a full report or only the messages -reports=yes +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= +# Naming style matching correct class names. +class-naming-style=PascalCase +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= -[BASIC] +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,input +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +# Naming style matching correct method names. +method-naming-style=snake_case -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ -# Regular expression matching correct function names -function-rgx=[A-Za-z_][A-Za-z0-9_]{2,30}$ +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Naming style matching correct variable names. +variable-naming-style=snake_case -# Regular expression matching correct constant names -const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ +[CLASSES] -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ +[DESIGN] -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of arguments for function / method. +max-args=5 -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of attributes for a class (see R0902). +max-attributes=7 -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of branch for function / method body. +max-branches=12 -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ +# Maximum number of locals for function / method body. +max-locals=15 -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 +# Maximum number of return / yield for function / method body. +max-returns=6 -[ELIF] +# Maximum number of statements in function / method body. +max-statements=50 -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] -# Maximum number of characters on a single line. -max-line-length=120 +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator -# Maximum number of lines in a module -max-module-lines=1000 +[IMPORTS] -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= [LOGGING] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + # Logging modules to check that the string format arguments are in logging -# function parameter format +# function parameter format. logging-modules=logging +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO +notes=FIXME, + XXX, + TODO +# Regular expression of note tags to take in consideration. +notes-rgx= -[SIMILARITIES] -# Minimum lines number of a similarity. -min-similarity-lines=4 +[REFACTORING] -# Ignore comments when computing similarities. -ignore-comments=yes +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 -# Ignore docstrings when computing similarities. -ignore-docstrings=yes +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error -# Ignore imports when computing similarities. -ignore-imports=no +[REPORTS] -[SPELLING] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= -# List of comma separated words that should not be checked. -spelling-ignore-words= +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= +# Tells whether to display a full report or only the messages. +reports=no -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no +# Activate the evaluation score. +score=yes -[TYPECHECK] +[SIMILARITIES] -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes +# Comments are removed from the similarity computation +ignore-comments=yes -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= +# Docstrings are removed from the similarity computation +ignore-docstrings=yes -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). This supports can work -# with qualified names. -# -# avoids the following error: -# -# pylint -E ./check_zookeeper_version.py -# -# E: 69,12: Instance of '_socketobject' has no 'sendall' member (no-member) -# E: 70,19: Instance of '_socketobject' has no 'recv' member (no-member) -# -ignored-classes=SQLObject,_socketobject +# Imports are removed from the similarity computation +ignore-imports=yes -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= +# Signatures are removed from the similarity computation +ignore-signatures=yes +# Minimum lines number of a similarity. +min-similarity-lines=4 -[VARIABLES] -# Tells whether we should check for unused import in __init__ files. -init-import=no +[SPELLING] -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: +# List of comma separated words that should not be checked. +spelling-ignore-words= -[CLASSES] +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs +[STRING] -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no -[DESIGN] -# Maximum number of arguments for function / method -max-args=5 +[TYPECHECK] -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager -# Maximum number of locals for function / method body -max-locals=15 +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= -# Maximum number of return / yield for function / method body -max-returns=6 +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes -# Maximum number of branch for function / method body -max-branches=12 +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes -# Maximum number of statements in function / method body -max-statements=50 +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init -# Maximum number of parents for a class (see R0901). -max-parents=7 +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace -# Maximum number of attributes for a class (see R0902). -max-attributes=7 +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin +# List of decorators that change the signature of a decorated function. +signature-mutators= -[IMPORTS] -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec +[VARIABLES] -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= +# List of names allowed to shadow builtins +allowed-redefined-builtins= +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb -[EXCEPTIONS] +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io From 64b326acbcea84535f033b706e6bfb5b289a5f32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 03:29:52 +0100 Subject: [PATCH 215/459] updated README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index edb49e6b1..f064e78b0 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ [![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Codiga Grade](https://api.codiga.io/project/8839/status/svg)](https://app.codiga.io/project/8839/dashboard) -[![Codiga Score](https://api.codiga.io/project/8839/score/svg)](https://app.codiga.io/project/8839/dashboard) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) From 0487abce888a1fdd84a18f853ef1871e17772147 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 216/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 37dede02a..09a50e7e1 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 37dede02a5a184087798740bfbb36b90cf5ed685 +Subproject commit 09a50e7e1810c856eff735279a83c00116a7fcd5 From 493c6d1afb562ca7eca054bde3d48fd90b9586c0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 217/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5f2f181a9..c3ab2659f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5f2f181a96f884acd0340125aab776414c343b68 +Subproject commit c3ab2659f369a3af7ee16344ef52e222fcd7a555 From a0fed602600e01b0aa97e35e6824560e71de1bb9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 17:32:12 +0100 Subject: [PATCH 218/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 8d838daa6..0d8bd4872 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 8d838daa669dc05d497d5fb2fa1aef631c613acf +Subproject commit 0d8bd48723672bbfec6eb549c80e564b28725a7b From dd64db0434d294fc3bd53a2858e4da7920667009 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:04:10 +0100 Subject: [PATCH 219/459] added kics.config --- kics.config | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 kics.config diff --git a/kics.config b/kics.config new file mode 100644 index 000000000..5fafd16d2 --- /dev/null +++ b/kics.config @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-05 18:05:53 +0100 (Fri, 05 May 2023) +# +# vim:ts=2:sts=2:sw=2:et:filetype=yaml +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# K i c s C o n f i g +# ============================================================================ # + +# https://github.com/Checkmarx/kics/blob/master/docs/configuration-file.md + +--- +#path: assets/iac_samples +verbose: true +log-file: true +#type: +# - Dockerfile +# - Kubernetes +#queries-path: "assets/queries" +exclude-paths: + # ignore submodules - handle them in the source repos only + - bash-tools/ + - pylib/ + - sql/ + - templates/ +#output-path: "results" From fee21a0c50c83186949d1ae3253f0882ae642054 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 220/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 09a50e7e1..b2bcc4c93 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 09a50e7e1810c856eff735279a83c00116a7fcd5 +Subproject commit b2bcc4c93fcfc3e89421d8d264e6a863d623cf32 From 131da36d872ec1e3b2013de15589703b594f5323 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 221/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c3ab2659f..e137295ce 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c3ab2659f369a3af7ee16344ef52e222fcd7a555 +Subproject commit e137295ce7ae4c484c705a7e800c3577af823cef From 0ba9caa0aa08f5e044c5e340b195ad008944fb1e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 19:22:19 +0100 Subject: [PATCH 222/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 0d8bd4872..73d5f3b1f 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 0d8bd48723672bbfec6eb549c80e564b28725a7b +Subproject commit 73d5f3b1fa54a3900493417637652f057fab3b61 From 5c929d22aa11b8553ec809a1b11172b933047ec2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 21:54:57 +0100 Subject: [PATCH 223/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b2bcc4c93..9112b3d95 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b2bcc4c93fcfc3e89421d8d264e6a863d623cf32 +Subproject commit 9112b3d957ce48300a139fc94c1da963c2fa8bf0 From 62046edaf17c3d87f6e7d1f281813e0ec1b2aaf2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 21:54:57 +0100 Subject: [PATCH 224/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e137295ce..6a6b9cf72 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e137295ce7ae4c484c705a7e800c3577af823cef +Subproject commit 6a6b9cf7211121fa2bb3d8f72dd1ef8ba19bcdf5 From 6f96ebf189863beae3e9612a6deb2743044dee24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 5 May 2023 22:51:14 +0100 Subject: [PATCH 225/459] updated README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f064e78b0..9c78001bb 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,8 @@ [![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) [![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) From afe9d7a35dc370d604111a9e15c824e4d63f6fe2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:48 +0100 Subject: [PATCH 226/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9112b3d95..a3750afbd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9112b3d957ce48300a139fc94c1da963c2fa8bf0 +Subproject commit a3750afbd00c67a0c497367be19bd807f05c242b From c9ac6b65bb38bbb6d5fa2375b5810d03fa9e46a9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:48 +0100 Subject: [PATCH 227/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 6a6b9cf72..5f2c84abb 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 6a6b9cf7211121fa2bb3d8f72dd1ef8ba19bcdf5 +Subproject commit 5f2c84abb142dfa218cc29fab89d11ddffa2011f From b2d609d7192f656094daf38192d4a396e5afee3a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:20:49 +0100 Subject: [PATCH 228/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 73d5f3b1f..590fd1816 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 73d5f3b1fa54a3900493417637652f057fab3b61 +Subproject commit 590fd18169dfed69dd5861d1b33d9c23b4a71a03 From 0341c77f80096ce8a3c7171d5e92ced882d973e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:25:24 +0100 Subject: [PATCH 229/459] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 9c78001bb..2ccdff67a 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,11 @@ [![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) [![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/HariSekhon/DevOps-Python-tools.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) From 472331bac11a4d00a39cb865dcc41540e3cdbc8d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:29:06 +0100 Subject: [PATCH 230/459] updated .pylintrc --- .pylintrc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 1c56d5de1..b35a5fccc 100644 --- a/.pylintrc +++ b/.pylintrc @@ -448,7 +448,10 @@ disable=raw-checker-failed, suppressed-message, useless-suppression, deprecated-pragma, - use-symbolic-message-instead + use-symbolic-message-instead, + missing-function-docstring, + super-with-arguments, + consider-using-f-string # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 93b2c4f3353810ae7a53f4b6707ec9e1c371b7a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 02:30:27 +0100 Subject: [PATCH 231/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 5f2c84abb..0f48c7572 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 5f2c84abb142dfa218cc29fab89d11ddffa2011f +Subproject commit 0f48c7572c549bb109c3d946c7908048854dec73 From 2c42d10e551497e571f0297b36636f04bfb1025a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:45:22 +0100 Subject: [PATCH 232/459] updated .pylintrc --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index b35a5fccc..2006619da 100644 --- a/.pylintrc +++ b/.pylintrc @@ -449,6 +449,7 @@ disable=raw-checker-failed, useless-suppression, deprecated-pragma, use-symbolic-message-instead, + missing-class-docstring, missing-function-docstring, super-with-arguments, consider-using-f-string From 7a42f40f3dfa0d0a02658d82f630a59d3f5d5e8b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:47:54 +0100 Subject: [PATCH 233/459] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index fb56e18cf..93ea98595 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -166,7 +166,6 @@ def create_struct(arg): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') @@ -198,5 +197,6 @@ def create_struct(arg): # the databricks avro driver df.write.format('com.databricks.spark.avro').save(avro_dir) + if __name__ == '__main__': SparkCSVToAvro().main() From a282dda7d3eb16d5d8777db108bec344e3d90fff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:43 +0100 Subject: [PATCH 234/459] updated spark_avro_to_parquet.py --- spark_avro_to_parquet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py index 9300c4fd1..8528af1a6 100755 --- a/spark_avro_to_parquet.py +++ b/spark_avro_to_parquet.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 246d4df8036306782234ed6c1c4d47a7dee9d44e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:43 +0100 Subject: [PATCH 235/459] updated spark_csv_to_avro.py --- spark_csv_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index 93ea98595..d41717b4f 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -57,6 +57,7 @@ 'com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From dad1bce7ddc6cc5972ba5df65df8ff0138b1f659 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 236/459] updated spark_csv_to_parquet.py --- spark_csv_to_parquet.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py index 8e3b0da1c..e70f014f6 100755 --- a/spark_csv_to_parquet.py +++ b/spark_csv_to_parquet.py @@ -49,6 +49,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-csv_2.11:1.5.0 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error @@ -157,9 +158,7 @@ def create_struct(arg): if not isVersionLax(spark_version): die("Spark version couldn't be determined. " + support_msg('pytools')) - # pylint: disable=invalid-name - - df = None + # pylint: disable=invalid-name if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') From 49c66be7fd0cc50d196ba7e6c1f0a503c14c1771 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 237/459] updated spark_json_to_avro.py --- spark_json_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index d53a4bffa..db9d069ed 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 7fb4da436c7016412fe3b7426ab4ea06b676e2e1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 238/459] updated spark_json_to_parquet.py --- spark_json_to_parquet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py index 2ed935917..6976ce153 100755 --- a/spark_json_to_parquet.py +++ b/spark_json_to_parquet.py @@ -43,6 +43,7 @@ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) sys.exit(4) pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From 86727ce5e22f53fa4d84627865c4cad83dddf70e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 6 May 2023 03:49:44 +0100 Subject: [PATCH 239/459] updated spark_parquet_to_avro.py --- spark_parquet_to_avro.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py index 0186981f5..030816963 100755 --- a/spark_parquet_to_avro.py +++ b/spark_parquet_to_avro.py @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error From ec22d9ca18fbd0ec75d1137a2ae11c8001247286 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:50 +0100 Subject: [PATCH 240/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a3750afbd..1b0b1149f 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a3750afbd00c67a0c497367be19bd807f05c242b +Subproject commit 1b0b1149fabbbdb4a1c07cd25d2c1941034eb0a1 From f81d2e16df704a4fee44f20a3a48658b0e1f7c17 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:51 +0100 Subject: [PATCH 241/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 0f48c7572..e53f962a7 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 0f48c7572c549bb109c3d946c7908048854dec73 +Subproject commit e53f962a7d0eee7cf1f89ca98552d90c0d6226cb From 1643e01fddfd93d2c301a9a6ca925c1192389ef2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 01:57:51 +0100 Subject: [PATCH 242/459] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 590fd1816..3a74368ea 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 590fd18169dfed69dd5861d1b33d9c23b4a71a03 +Subproject commit 3a74368ead1544983ac21af7500b380b9fc20b9f From dbc5cec14dea46ef3f851d56988ab9173f8ace9e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:01 +0100 Subject: [PATCH 243/459] updated Makefile --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index a67a168a9..13e4c9b27 100644 --- a/Makefile +++ b/Makefile @@ -86,12 +86,12 @@ init: # @$(MAKE) $@c %.pyc:: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing - @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ echo Generated $@ %.pyo:: %.py - @PIP=$(PIP) bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile $< && \ echo && \ echo Generated $@ @@ -118,15 +118,15 @@ python: pylib @# only install pip packages not installed via system packages @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt @#$(SUDO_PIP) $(PIP) install -r requirements.txt - @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt + @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3"; fi - PIP=$(PIP) bash-tools/python_pip_install.sh "avro-python3" || : + @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3"; fi + PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3" || : @# for impyla @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || : @@ -154,7 +154,7 @@ python: pylib @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi @# Python >= 2.7 - won't build on 2.6, handle separately and accept failure - @PIP=$(PIP) bash-tools/python_pip_install.sh "ipython[notebook]" || : + @PIP=$(PIP) bash-tools/python/python_pip_install.sh "ipython[notebook]" || : @echo $(MAKE) pycompile @echo @@ -216,7 +216,7 @@ test: .PHONY: basic-test basic-test: test-lib - bash-tools/check_all.sh + bash-tools/checks/check_all.sh .PHONY: install install: build From bf4934fe8873bc1153ac2fef4f796a112fc93483 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:38 +0100 Subject: [PATCH 244/459] updated all.sh --- tests/all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/all.sh b/tests/all.sh index 33452eb7c..dbac7b8fb 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -31,7 +31,7 @@ section "Running PyTools ALL" cd "$srcdir/.."; # shellcheck disable=SC1090 # has to be included so that isExcluded function is inherited -. "$srcdir/../bash-tools/check_all.sh" +. "$srcdir/../bash-tools/checks/check_all.sh" #tests/test_yamllint.sh @@ -41,4 +41,4 @@ exit 0 # pyspark not found tests/help.sh -bash-tools/run_tests.sh +bash-tools/checks/run_tests.sh From 9a1f9be8ea238a1bc7cb04291f1eb743308bc09b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 10 May 2023 02:00:47 +0100 Subject: [PATCH 245/459] updated anonymize_parallel.sh --- anonymize_parallel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh index 1f34f863a..abb7e9d21 100755 --- a/anonymize_parallel.sh +++ b/anonymize_parallel.sh @@ -68,7 +68,7 @@ for filename in $file_list; do echo "Removing any pre-existing parts:" rm -v "$filename".* 2>/dev/null || : echo - "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename" + "$srcdir/bash-tools/bin/split.sh" --parts "$parallelism" "$filename" echo "Anonymizing parts" for file_part in "$filename".*; do cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized" From 7251c68560d2f1f605475a8fb0df98e28c18353f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 11 May 2023 23:02:38 +0100 Subject: [PATCH 246/459] updated sonar-project.properties --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 270ec8ac7..7189b1a16 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -21,7 +21,7 @@ sonar.projectDescription=Python / Jython Tools sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues -sonar.links.ci=https://travis-ci.org/HariSekhon/devops-python-tools +sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions sonar.sources=. From 754947957d53ef5ed572bdcbb2b91a6f4f4b583c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 11 May 2023 23:59:10 +0100 Subject: [PATCH 247/459] updated README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2ccdff67a..56a02a41e 100644 --- a/README.md +++ b/README.md @@ -469,6 +469,10 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery +- [Jenkins](https://github.com/HariSekhon/Jenkins) - Advanced Jenkinsfile & Jenkins Groovy Shared Library + +- [GitHub-Actions](https://github.com/HariSekhon/GitHub-Actions) - GitHub Actions master template & GitHub Actions Shared Workflows library + - [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. - [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments From 01af37704aed0bcbe5eeb78d0bbcd3e849358b63 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 00:37:00 +0100 Subject: [PATCH 248/459] added trivy.yaml --- .github/workflows/trivy.yaml | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/trivy.yaml diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 000000000..5c9475308 --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# T r i v y +# ============================================================================ # + +--- +name: Trivy + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + trivy: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Trivy Filesystem Scan + uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From 0d560dd9f6634ad3ef668f2aa8ee52ab20d0f204 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 00:47:26 +0100 Subject: [PATCH 249/459] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 56a02a41e..aeab7f613 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) +[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) -[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) From 585e9050a905151a82746a4a8bfe42b993bf7ed6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 01:08:50 +0100 Subject: [PATCH 250/459] updated trivy.yaml --- .github/workflows/trivy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 5c9475308..8af2aeb39 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -1,6 +1,6 @@ # # Author: Hari Sekhon -# Date: Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) # # vim:ts=2:sts=2:sw=2:et # @@ -54,7 +54,7 @@ jobs: # github.event.repository context not available in scheduled workflows #if: github.event.repository.fork == false if: github.repository_owner == 'HariSekhon' - name: Trivy Filesystem Scan + name: Trivy uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master with: debug: ${{ github.event.inputs.debug }} From 92b23f826de7fcdab498eb6b9e0ac26b306f09d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 01:41:06 +0100 Subject: [PATCH 251/459] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index aeab7f613..a41120672 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) [![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) From 9fe0d3ca28260ec2c4414a6f77c6ebbd0e5c390f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 13 May 2023 07:15:53 +0100 Subject: [PATCH 252/459] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a41120672..36f322455 100644 --- a/README.md +++ b/README.md @@ -467,7 +467,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu ## Related Repositories -- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 800+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 1000+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies - [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From b8c21da103ec580fb9b26ade93ad3eeb664cd2db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 14 May 2023 04:18:31 +0100 Subject: [PATCH 253/459] added grype.yaml --- .github/workflows/grype.yaml | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/grype.yaml diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml new file mode 100644 index 000000000..6a321f86e --- /dev/null +++ b/.github/workflows/grype.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G r y p e +# ============================================================================ # + +--- +name: Grype + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + Grype: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Grype + uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master + with: + debug: ${{ github.event.inputs.debug }} From a8c29b46dd6d8b0659fb64bbbc41c1d2cd6666e9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 29 May 2023 02:06:58 +0100 Subject: [PATCH 254/459] updated center.py --- center.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/center.py b/center.py index b9cbac5f1..379353e68 100755 --- a/center.py +++ b/center.py @@ -44,7 +44,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.5.1' class Center(CLI): @@ -104,7 +104,7 @@ def process_line(self, line): char = ' ' # preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it line = line.strip() - if isChars(line[0], '#'): + if line and isChars(line[0], '#'): char = line[0] line = line.lstrip(char) elif len(line) > 1 and isChars(line[0:1], '/'): From f9abc5f8321e73b34e0be59079823ce850b50362 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:24 +0100 Subject: [PATCH 255/459] removed codeship.yml --- codeship.yml | 58 ---------------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 codeship.yml diff --git a/codeship.yml b/codeship.yml deleted file mode 100644 index f466bd6a7..000000000 --- a/codeship.yml +++ /dev/null @@ -1,58 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2021-04-12 18:33:44 +0100 (Mon, 12 Apr 2021) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# ============================================================================ # -# C o d e S h i p -# ============================================================================ # - -# LEGACY - CloudBees has removed free plans as of 1st March 2023 - -# 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this - -# https://github.com/painless-software/codeship-yaml - -# Requires setting up the CodeShip commands like so: -# -# pip install codeship-yaml -# codeship-yaml -# -# or seaparately in sections: -# -# Project Settings > Test Settings > Setup Commands: -# -# pip install codeship-yaml -# codeship-yaml install -# -# Project Settings > Test Settings > Test Commands: -# -# codeship-yaml before_script script -# -# Project Settings > Deployment > (branch name) -# -# codeship-yaml after_success - ---- -install: - # these cause package installation breakages due to GPG or 403 errors, old addresses etc. - - sudo rm -fv -- /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list - - sudo rm -fv -- /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list - - sudo rm -fv -- /etc/apt/sources.list.d/www_apache_org_dist_cassandra_debian.list - - make -#before_script: -# - somecommand -script: - - make test -#after_success: -# - echo "Now we can deploy" From c4c513b66715f87bbed88320deda71205f84ce96 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:32 +0100 Subject: [PATCH 256/459] removed wercker.yml --- wercker.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 wercker.yml diff --git a/wercker.yml b/wercker.yml deleted file mode 100644 index c27be30ed..000000000 --- a/wercker.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# https://devcenter.wercker.com/reference/wercker-yml/ - -box: debian - -build: - steps: - - script: - name: ci bootstrap - code: setup/ci_bootstrap.sh - - script: - name: init - code: make init - - script: - name: build - code: make ci - - script: - name: test - code: make test From 116cdfda1b85dbd8803e21a3e126c1d4d58d8ab6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:01:38 +0100 Subject: [PATCH 257/459] removed .werckerignore --- .werckerignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .werckerignore diff --git a/.werckerignore b/.werckerignore deleted file mode 100644 index ff8fb8247..000000000 --- a/.werckerignore +++ /dev/null @@ -1 +0,0 @@ -**/.md From 5ba24dc535c907d2d574d652da6517e3ec2cf75a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:08 +0100 Subject: [PATCH 258/459] moved .teamcity.vcs.json to teamcity/ --- .teamcity.vcs.json => teamcity/.teamcity.vcs.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.json => teamcity/.teamcity.vcs.json (100%) diff --git a/.teamcity.vcs.json b/teamcity/.teamcity.vcs.json similarity index 100% rename from .teamcity.vcs.json rename to teamcity/.teamcity.vcs.json From 1730b699555891aff16137749cfbd88f7559f21b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:09 +0100 Subject: [PATCH 259/459] moved .teamcity.vcs.oauth.json to teamcity/ --- .teamcity.vcs.oauth.json => teamcity/.teamcity.vcs.oauth.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.oauth.json => teamcity/.teamcity.vcs.oauth.json (100%) diff --git a/.teamcity.vcs.oauth.json b/teamcity/.teamcity.vcs.oauth.json similarity index 100% rename from .teamcity.vcs.oauth.json rename to teamcity/.teamcity.vcs.oauth.json From 21d86fe66caa9b86030bedd0155287ad263ba17f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:03:09 +0100 Subject: [PATCH 260/459] moved .teamcity.vcs.ssh.json to teamcity/ --- .teamcity.vcs.ssh.json => teamcity/.teamcity.vcs.ssh.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .teamcity.vcs.ssh.json => teamcity/.teamcity.vcs.ssh.json (100%) diff --git a/.teamcity.vcs.ssh.json b/teamcity/.teamcity.vcs.ssh.json similarity index 100% rename from .teamcity.vcs.ssh.json rename to teamcity/.teamcity.vcs.ssh.json From 637c5ade2863627f66f26bd6ee354416c698e69d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 261/459] moved .concourse.yml to cicd/ --- .concourse.yml => cicd/.concourse.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .concourse.yml => cicd/.concourse.yml (100%) diff --git a/.concourse.yml b/cicd/.concourse.yml similarity index 100% rename from .concourse.yml rename to cicd/.concourse.yml From 424a095fedc3298d2ce34a63f6a4296b2acb8140 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 262/459] moved .gocd.yml to cicd/ --- .gocd.yml => cicd/.gocd.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .gocd.yml => cicd/.gocd.yml (100%) diff --git a/.gocd.yml b/cicd/.gocd.yml similarity index 100% rename from .gocd.yml rename to cicd/.gocd.yml From 2451e74fd432d4cdac907970369e2c9059608f3f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 263/459] moved buildspec.yml to cicd/ --- buildspec.yml => cicd/buildspec.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename buildspec.yml => cicd/buildspec.yml (100%) diff --git a/buildspec.yml b/cicd/buildspec.yml similarity index 100% rename from buildspec.yml rename to cicd/buildspec.yml From fca148ae783746b8452a26818344156c5a1bcdae Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:05:11 +0100 Subject: [PATCH 264/459] moved cloudbuild.yaml to cicd/ --- cloudbuild.yaml => cicd/cloudbuild.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cloudbuild.yaml => cicd/cloudbuild.yaml (100%) diff --git a/cloudbuild.yaml b/cicd/cloudbuild.yaml similarity index 100% rename from cloudbuild.yaml rename to cicd/cloudbuild.yaml From caf90ee899ba2b558bdbab7d28c87ae28c5da7ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:41:46 +0100 Subject: [PATCH 265/459] removed shippable.yml --- shippable.yml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 shippable.yml diff --git a/shippable.yml b/shippable.yml deleted file mode 100644 index 1fa06bc06..000000000 --- a/shippable.yml +++ /dev/null @@ -1,50 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# http://docs.shippable.com/platform/workflow/config/ - -# http://docs.shippable.com/ci/advancedOptions/environmentVariables/ - -language: none - -branches: - only: - - master - -build: - ci: - # workaround to broken repos - # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB - # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 - # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed. - # DevOps-Python-tools/Makefile.in:272: recipe for target 'apt-packages' failed - - rm -fv -- /etc/apt/sources.list.d/cassandra.sources.list* - - rm -fv -- /etc/apt/sources.list.d/yarn.list* - # Basho repo is giving a '402 payment required' error - # https://github.com/Shippable/support/issues/5172 - - rm -fv -- /etc/apt/sources.list.d/basho_riak.list - #- shippable_retry make - - setup/ci_bootstrap.sh - - make init - - make ci - - make test - -integrations: - notifications: - - integrationName: email - type: email - on_success: never - on_failure: never - on_pull_request: never From 7d04cce80786cc6c4f278ce4a773f7d6a6f69ae3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:01 +0100 Subject: [PATCH 266/459] updated .appveyor.yml --- .appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index f4b36fba0..c459fd55a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A p p V e y o r C I +# ============================================================================ # + # https://www.appveyor.com/docs/appveyor-yml/ image: Ubuntu From 00b3cf9ae996e7b5a8f5bc7577adfd9b420b9381 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:02 +0100 Subject: [PATCH 267/459] updated pipeline.yml --- .buildkite/pipeline.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 93f047c02..eb0d641fd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u i l d K i t e C I +# ============================================================================ # + # BuildKite Pipeline # # add this command to the UI and it will read the rest of the steps from here: From 526798d5606da7d44e405fb9931b9d27c9c39b71 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:03 +0100 Subject: [PATCH 268/459] updated config.yml --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5dbde36b7..09026fc6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r c l e C I +# ============================================================================ # + # Master Template with more advanced config: # # https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml From 126ad67b87f9f2ada233929556a37d3ee479126e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:04 +0100 Subject: [PATCH 269/459] updated .cirrus.yml --- .cirrus.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index d544aeb85..6916e2577 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r r u s C I +# ============================================================================ # + # https://cirrus-ci.org/guide/writing-tasks/ container: From 243482c43161216eb1c278c64a0451f731347385 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:05 +0100 Subject: [PATCH 270/459] updated .drone.yml --- .drone.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.drone.yml b/.drone.yml index 4d36c9db2..d105ab57a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,6 +15,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# D r o n e C I +# ============================================================================ # + # https://docs.drone.io/quickstart/cli/ # # https://docs.drone.io/cli/install/ From 84d0b04dff32333bb7da8058de7c36b98bab6015 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 02:50:06 +0100 Subject: [PATCH 271/459] updated .gitlab-ci.yml --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 477b32c3e..625220b32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G i t L a b C I +# ============================================================================ # + # https://docs.gitlab.com/ee/ci/yaml/README.html +#include: '.gitlab/*.y*ml' + image: ubuntu:18.04 job: From c21088d44d170cf28533c2af5633c753b3a493ff Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:15:38 +0100 Subject: [PATCH 272/459] updated sonar-project.properties --- sonar-project.properties | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/sonar-project.properties b/sonar-project.properties index 7189b1a16..9b1539430 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon -# Date: 2016-07-19 18:18:27 +0100 (Tue, 19 Jul 2016) +# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016) # # https://github.com/HariSekhon/DevOps-Python-tools # @@ -12,21 +12,30 @@ # https://www.linkedin.com/in/HariSekhon # -sonar.projectName=DevOps Python Tools -sonar.projectKey=pytools +# ============================================================================ # +# S o n a r Q u b e +# ============================================================================ # + +sonar.host.url=https://sonarcloud.io + +# Required metadata +sonar.organization=harisekhon +sonar.projectName=DevOps-Python-tools +sonar.projectKey=HariSekhon_DevOps-Python-tools sonar.projectVersion=1.0 -sonar.projectDescription=Python / Jython Tools +sonar.projectDescription=DevOps-Python-tools sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions +# directories to scan (defaults to sonar-project.properties dir otherwise) sonar.sources=. #sonar.language=py sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/spark*/**/* +sonar.exclusions=**/tests/** From 2c93d263101b61483ad2189406aefe87efe53ecf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:39 +0100 Subject: [PATCH 273/459] updated sonar-project.properties --- sonar-project.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 9b1539430..319cc6897 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -38,4 +38,5 @@ sonar.sources=. sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/** +#sonar.exclusions=**/tests/** +sonar.exclusions=**/zookeeper-*/**/* From 3b6c0b0324ba01c6596adb589c86ad6790225fbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:50 +0100 Subject: [PATCH 274/459] updated semaphore.yml --- .semaphore/semaphore.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e86c4f36e..761ef0697 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S e m a p h o r e C I +# ============================================================================ # + # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ version: v1.0 From 7e975e2cda81ac8b0177401bc8766caf7db9bf6c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:51 +0100 Subject: [PATCH 275/459] updated README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 36f322455..35a30a739 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://harisekhon.github.io/CI-CD/) [![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) -[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml) -[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.concourse.yml) +[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.gocd.yml) [![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) [![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) @@ -55,8 +55,8 @@ [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) [![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) [![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) -[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) -[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) +[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) +[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) [![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) From 97a95b048ddff13a76f31a2687f62a8c3f4ee10f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:52 +0100 Subject: [PATCH 276/459] updated azure-pipelines.yml --- azure-pipelines.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 76ad8f1f0..e26e7f760 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A z u r e D e v O p s P i p e l i n e +# ============================================================================ # + # https://aka.ms/yaml trigger: From e7b73f1e14052b5e152d8934fddd83e11929db02 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:53 +0100 Subject: [PATCH 277/459] updated buddy.yml --- buddy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/buddy.yml b/buddy.yml index f901a6425..e0b57451f 100644 --- a/buddy.yml +++ b/buddy.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u d d y C I +# ============================================================================ # + # https://buddy.works/docs/yaml/yaml-schema --- From 3f82815fe59aae0a425f870296787d9206c161c3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:16:59 +0100 Subject: [PATCH 278/459] updated .concourse.yml --- cicd/.concourse.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cicd/.concourse.yml b/cicd/.concourse.yml index 42b806f26..92847121e 100644 --- a/cicd/.concourse.yml +++ b/cicd/.concourse.yml @@ -13,6 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o n c o u r s e C I +# ============================================================================ # + +# https://concourse-ci.org/golang-library-example.html + +# https://resource-types.concourse-ci.org/ +# https://concourse-ci.org/resource-types.html resources: - name: github icon: github-circle @@ -25,8 +33,7 @@ resources: # source: # interval: 1d -# https://concourse-ci.org/golang-library-example.html - +# https://concourse-ci.org/jobs.html jobs: - name: build public: false From 3851ded922052a5aa98677a3045fcbca03a13aa3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:00 +0100 Subject: [PATCH 279/459] updated .gocd.yml --- cicd/.gocd.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cicd/.gocd.yml b/cicd/.gocd.yml index f3ed9fd19..65583a818 100644 --- a/cicd/.gocd.yml +++ b/cicd/.gocd.yml @@ -12,6 +12,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G o C D +# ============================================================================ # + # https://github.com/tomzo/gocd-yaml-config-plugin#setup # https://docs.gocd.org/current/configuration/configuration_reference.html From f7dda886aad2cdcd0374877ca6b5ab9dae4bd2d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:00 +0100 Subject: [PATCH 280/459] updated codefresh.yml --- codefresh.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codefresh.yml b/codefresh.yml index 5326f6e51..7cddcaf7c 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e f r e s h C I +# ============================================================================ # + # https://codefresh.io/docs/docs/codefresh-yaml/ version: "1.0" From 0865b14f7edd4a67a348af0139713a706fa67e3a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:01 +0100 Subject: [PATCH 281/459] updated kics.config --- kics.config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kics.config b/kics.config index 5fafd16d2..51ac5712e 100644 --- a/kics.config +++ b/kics.config @@ -30,7 +30,15 @@ log-file: true exclude-paths: # ignore submodules - handle them in the source repos only - bash-tools/ + - github-actions/ + - haproxy-configs/ + - jenkins/ + - kubernetes-templates/ + - lib/ - pylib/ + - spotify-tools/ - sql/ + - sql-keywords/ - templates/ + - terraform-templates/ #output-path: "results" From 0ba05249f4a515b998110738105078ef6cb4d003 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:17:02 +0100 Subject: [PATCH 282/459] updated gocd_config_repo.json --- setup/gocd_config_repo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json index 4c1639b4d..ce63bf63d 100644 --- a/setup/gocd_config_repo.json +++ b/setup/gocd_config_repo.json @@ -12,7 +12,7 @@ "configuration": [ { "key": "file_pattern", - "value": "*.gocd.y*ml" + "value": "cicd/*.gocd.y*ml" } ], "rules": [ From f605276d04c77181fa4e8cf71c85cf6002752a35 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:05 +0100 Subject: [PATCH 283/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1b0b1149f..36e0eabe2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1b0b1149fabbbdb4a1c07cd25d2c1941034eb0a1 +Subproject commit 36e0eabe2b02207f51cd807fa6dafd18abdab234 From 37b79941c2329f2de041cab714d08851c1eaa5f8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 284/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e53f962a7..af52e8795 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e53f962a7d0eee7cf1f89ca98552d90c0d6226cb +Subproject commit af52e87958755d74d9b96cd373aab8cc810ab6a6 From 0d7d8060f515735afca206920e275643934e4867 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 285/459] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index d4eedc0e2..b96c184fd 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit d4eedc0e2c68c9a43838c89a1d09abf10927a566 +Subproject commit b96c184fdda9b2ad607d222ec5663a087769b561 From d1ef74ffebc27e32ec9afeb5bca7debd4fedb64b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 03:25:06 +0100 Subject: [PATCH 286/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3a74368ea..d58140c42 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3a74368ead1544983ac21af7500b380b9fc20b9f +Subproject commit d58140c424a83a440fa1480e878c45a00920c0f4 From a3634ecf1e18a624af24814f4540cba15623a642 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:32 +0100 Subject: [PATCH 287/459] updated codeowners.yaml --- .github/workflows/codeowners.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 6b9eaf3b9..4dc3a09c3 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -11,8 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e O w n e r s +# ============================================================================ # + --- -name: Codeowners +name: CodeOwners on: push: From 79fdd3ef11e57c87b703255fc61789aedd083080 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:33 +0100 Subject: [PATCH 288/459] updated fork-sync.yaml --- .github/workflows/fork-sync.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 4d4ebd577..c7ff16fd4 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k S y n c +# ============================================================================ # + +# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch + --- name: Fork Sync From 6900c0b4ca31f052e94e7caffb77c16ec8dd7938 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:34 +0100 Subject: [PATCH 289/459] updated fork-update-pr.yaml --- .github/workflows/fork-update-pr.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 00479ecb7..94398083f 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -11,6 +11,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k U p d a t e P R +# ============================================================================ # + +# For a fork of the original repo, activate to keep its branches up to date via Pull Requests +# +# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date + --- name: Fork Update PR From 02e4b41f7117802742d8af57107da9aa09677982 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:35 +0100 Subject: [PATCH 290/459] updated json.yaml --- .github/workflows/json.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index 6ed4b92d1..f68e309e5 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# J S O N +# ============================================================================ # + +# Validate any JSON files found in the repo + --- name: JSON From 77f1608b618c75c87ad51006dfddba709aabdf37 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:35 +0100 Subject: [PATCH 291/459] updated kics.yaml --- .github/workflows/kics.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index 399cc40f0..033f4331e 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -11,6 +11,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# K i c s +# ============================================================================ # + --- name: Kics From 70a101b1ae1095ba49d05a182d3a77a7eb352f73 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:36 +0100 Subject: [PATCH 292/459] updated shellcheck.yaml --- .github/workflows/shellcheck.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 98e53bfe4..753a6a495 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S h e l l C h e c k +# ============================================================================ # + +# Validate any shell scripts found in the repo + --- name: ShellCheck From 55a9f677eb8afdb95879e413b15da0b3d5543b27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:36 +0100 Subject: [PATCH 293/459] updated trivy.yaml --- .github/workflows/trivy.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 8af2aeb39..26f01fa94 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -15,6 +15,8 @@ # T r i v y # ============================================================================ # +# Scan files in the local repo + --- name: Trivy From b5f76c8c6c099ca81c0e0d6885d41f1195d85379 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:37 +0100 Subject: [PATCH 294/459] updated validate.yaml --- .github/workflows/validate.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 5d924b77b..94a8c007a 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# V a l i d a t i o n +# ============================================================================ # + +# Run all custom validations against files in the repo + --- name: Validation From 88ac6b6661928c5f272515fe0700576d04b6f279 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:37 +0100 Subject: [PATCH 295/459] updated xml.yaml --- .github/workflows/xml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml index f4d3b7035..8b07ed256 100644 --- a/.github/workflows/xml.yaml +++ b/.github/workflows/xml.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# X M L +# ============================================================================ # + +# Validate any XML files found in the repo + --- name: XML From 2c992b85d84bb9dd9ebd7ea2e3ab57dfafe13ab2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:12:38 +0100 Subject: [PATCH 296/459] updated yaml.yaml --- .github/workflows/yaml.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index bee8be84c..d252f67d2 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -11,6 +11,12 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# Y A M L +# ============================================================================ # + +# Validate any YAML files found in the repo + --- name: YAML From a2a575ec7dec99394aab6b8572f42953129e3e43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:49 +0100 Subject: [PATCH 297/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 36e0eabe2..815d75082 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 36e0eabe2b02207f51cd807fa6dafd18abdab234 +Subproject commit 815d75082d4faa8494e9500727ebd13e2c5abcae From 202d2ed1a86e2bd72f131fdd262eb6b88a846132 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 298/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index af52e8795..ae8aa8753 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit af52e87958755d74d9b96cd373aab8cc810ab6a6 +Subproject commit ae8aa875359ddea2c9b5cf6ccc422e12a0740949 From 25abbfdedc8b9832c507f970f7c1a8ef29643a27 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 299/459] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index b96c184fd..e11486ebd 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit b96c184fdda9b2ad607d222ec5663a087769b561 +Subproject commit e11486ebd436802d22d1b88bda00c0a9eb3d41b6 From 24407a2473022336a734c95d262203780f75df87 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:18:50 +0100 Subject: [PATCH 300/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d58140c42..3af953c9a 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d58140c424a83a440fa1480e878c45a00920c0f4 +Subproject commit 3af953c9ae196d4ba5e59637e9668bd1bc6e9953 From 82da083e55bda834adf1802587856c6ff5b8b95c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:01 +0100 Subject: [PATCH 301/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 815d75082..38901a555 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 815d75082d4faa8494e9500727ebd13e2c5abcae +Subproject commit 38901a555d2ce114f0e7df6d4a31a3fdc1a3dc41 From 4b432be8807c49caa2682b2f04ad492d403f2f0a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:02 +0100 Subject: [PATCH 302/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index ae8aa8753..aa20abd72 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit ae8aa875359ddea2c9b5cf6ccc422e12a0740949 +Subproject commit aa20abd72ceef27723ad8c461f4d9321ad12e779 From e14414287476521c71efa1ca3cb1095d2886ca62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 04:26:02 +0100 Subject: [PATCH 303/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 3af953c9a..15d1e3bc8 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 3af953c9ae196d4ba5e59637e9668bd1bc6e9953 +Subproject commit 15d1e3bc8a594171b3c70ce0bf04979eceb8630a From baec6ac43c0a3bf03cd7d78bb35d9972d470c922 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 2 Jun 2023 17:46:12 +0100 Subject: [PATCH 304/459] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35a30a739..eb8e5fe12 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,7 @@ Patches, improvements and even general feedback are welcome in the form of GitHu - [Perl Lib](https://github.com/HariSekhon/lib) - Perl version of above library -- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, ArgoCD, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... +- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python & D2 source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, Jenkins, ArgoCD, Traefik, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... From b5db3cb80db2ddc5dcc980a844939766452dc308 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:32:47 +0400 Subject: [PATCH 338/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index da278a058..1438f07bd 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit da278a05823c78709d831a4e6ba9cce4e5a481a7 +Subproject commit 1438f07bd9fe762752cfd986718e30797cff5650 From 8e50ccbc1c4ec668a1cea00152de7c700b584d24 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:32:47 +0400 Subject: [PATCH 339/459] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a43798fe9..6f9e99389 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a43798fe986928bffe4fe5952efdce408d012562 +Subproject commit 6f9e993896910c320c571918a635dc049e757e73 From a2d58fac6ac19b0d1b5f663aab5939eefe2b9f28 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 24 Apr 2024 12:33:56 +0400 Subject: [PATCH 340/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 1438f07bd..b69edeada 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1438f07bd9fe762752cfd986718e30797cff5650 +Subproject commit b69edeadadf0774fc6444c552f3a89d407ca24d4 From 39a4591fa22b1a6c0ffe94e0f8451fa3a5a60f1a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 27 May 2024 21:55:51 +0400 Subject: [PATCH 341/459] added .envrc --- .envrc | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..6fa553de2 --- /dev/null +++ b/.envrc @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# D i r e n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +# See more .envrc files in: +# +# https://github.com/HariSekhon/DevOps-Bash-tools +# +# .envrc-aws +# .envrc-gcp +# .envrc-kubernetes + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export COMPOSE_PROJECT_NAME="pytools" + +venv="$PWD/venv" + +if [ -f "$venv/bin/activate" ]; then + echo + echo "Local virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + echo +fi + +# read .env too +#dotenv From a0d2bc4abf677d283f992db3bab72461e888943f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:19:34 +0200 Subject: [PATCH 342/459] updated .envrc --- .envrc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.envrc b/.envrc index 6fa553de2..96d3b5c0f 100644 --- a/.envrc +++ b/.envrc @@ -35,18 +35,18 @@ set -euo pipefail export COMPOSE_PROJECT_NAME="pytools" -venv="$PWD/venv" - -if [ -f "$venv/bin/activate" ]; then - echo - echo "Local virtualenv directory found in: $venv" - echo - echo "Activating Virtualenv inside the directory: $venv" - - # shellcheck disable=SC1091 - source "$venv/bin/activate" - echo -fi +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + echo + fi +done # read .env too #dotenv From 0f9f99ff568ae21d903ceaaf0bf77b65c93632cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:20:41 +0200 Subject: [PATCH 343/459] updated .envrc --- .envrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.envrc b/.envrc index 96d3b5c0f..2aa9a4379 100644 --- a/.envrc +++ b/.envrc @@ -35,6 +35,7 @@ set -euo pipefail export COMPOSE_PROJECT_NAME="pytools" +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default for venv in "$PWD/venv" "$HOME/venv"; do if [ -f "$venv/bin/activate" ]; then echo From 92451e77812ddd6ace3c3ac401887fbeb2e1ad89 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:30 +0200 Subject: [PATCH 344/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index b69edeada..4cf683ab7 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit b69edeadadf0774fc6444c552f3a89d407ca24d4 +Subproject commit 4cf683ab744f849c34d2c15ce255b18b368d6a11 From eef5f927cf6bc15c6d438706a37567529347dd7d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:31 +0200 Subject: [PATCH 345/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 99f5c866a..804fcd30f 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 99f5c866aa5986c62d45de3b8da7043a28a63321 +Subproject commit 804fcd30fc9d05bf66fedae6001dbdef02dc8518 From 9616644dba9bf8d67673bcf02ae431af2395579d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 4 Jun 2024 09:52:31 +0200 Subject: [PATCH 346/459] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6f9e99389..a43798fe9 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6f9e993896910c320c571918a635dc049e757e73 +Subproject commit a43798fe986928bffe4fe5952efdce408d012562 From 7dd10775f06d708e16930d91519297addaa33519 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 347/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4cf683ab7..8e8cf26f2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4cf683ab744f849c34d2c15ce255b18b368d6a11 +Subproject commit 8e8cf26f2c49b30371a35bbd844bae491fccfaf4 From aa3ef778e851dd980c962b4fafe2600dffd50123 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 348/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 804fcd30f..c88505e98 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 804fcd30fc9d05bf66fedae6001dbdef02dc8518 +Subproject commit c88505e98b4a59ffa961ac92a581be96233d3b9d From 749724f6c95f2baba13490258a14b79b45c79f2f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:34:59 +0200 Subject: [PATCH 349/459] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index a43798fe9..c2521be52 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit a43798fe986928bffe4fe5952efdce408d012562 +Subproject commit c2521be525b4047222adb18263e5714e4171c483 From d8cbf436f9842d01a3bda55fdb095b2a4209b50d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:36:09 +0200 Subject: [PATCH 350/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c88505e98..545ce8b83 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c88505e98b4a59ffa961ac92a581be96233d3b9d +Subproject commit 545ce8b838428cb8e3ae8e883130a5fe251bd83d From 21f534993310e30479c38661de27c0ffc350946e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:38:47 +0200 Subject: [PATCH 351/459] updated .envrc --- .envrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.envrc b/.envrc index 2aa9a4379..9073c82fd 100644 --- a/.envrc +++ b/.envrc @@ -14,7 +14,7 @@ # # ============================================================================ # -# D i r e n v +# P y t h o n D i r E n v # ============================================================================ # # .envrc to auto-load the virtualenv inside the 'venv' directory if present From 82154119348fccba6d13f674c336bdc127c4ac44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:39:28 +0200 Subject: [PATCH 352/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8e8cf26f2..6d8f6ebc0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8e8cf26f2c49b30371a35bbd844bae491fccfaf4 +Subproject commit 6d8f6ebc0c93dad1306f35844b931c635cd1f9ce From 899a4cea79947f03e8e0862345914ddf47e68bee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 14:39:28 +0200 Subject: [PATCH 353/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 545ce8b83..76e9e28d1 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 545ce8b838428cb8e3ae8e883130a5fe251bd83d +Subproject commit 76e9e28d1b3a016dc64d1ccf92c7a967a74c48aa From a63287671e5471c97794e9c19f41b676d3d12e81 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 354/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6d8f6ebc0..9f1a8c533 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6d8f6ebc0c93dad1306f35844b931c635cd1f9ce +Subproject commit 9f1a8c53368a47e43e6f022f236e40635b78aa8f From 18cc287778458c4bfcaea0c12fca330302e6982c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 355/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 76e9e28d1..e2982d2f4 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 76e9e28d1b3a016dc64d1ccf92c7a967a74c48aa +Subproject commit e2982d2f4c07d0252d59897d9002235f6c0a7f6c From 1f6f01941062e293ee1c8898e884bafca470dccc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 356/459] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 39d0eccb4..8597f692e 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 39d0eccb4c160fcee9f09df9bb53c911ca2a387f +Subproject commit 8597f692e4e4abd59b58fb24c7fa79449a75c8c0 From e805b2ab2933e6a05cfb0fbd0f6dfbabf5e50c3b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 26 Jun 2024 19:03:58 +0200 Subject: [PATCH 357/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index c2521be52..5174a5073 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit c2521be525b4047222adb18263e5714e4171c483 +Subproject commit 5174a5073dbe25c8e5b1ac603b7b4f61b6ac7623 From 86d645a4f49915bd3032fcc9fee866f0a437eb94 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:40:47 +0200 Subject: [PATCH 358/459] updated .envrc --- .envrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.envrc b/.envrc index 9073c82fd..51de39abb 100644 --- a/.envrc +++ b/.envrc @@ -46,6 +46,7 @@ for venv in "$PWD/venv" "$HOME/venv"; do # shellcheck disable=SC1091 source "$venv/bin/activate" echo + break fi done From c6fcf3037124d84f6f0b9ce569993fe41eb76181 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:41:19 +0200 Subject: [PATCH 359/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 9f1a8c533..f32bb01be 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 9f1a8c53368a47e43e6f022f236e40635b78aa8f +Subproject commit f32bb01be9e59606bf4a292c0a3a7e2ef1ac4540 From 15816a21b449f0af739eb5985b61fe7ab9f9f475 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 16:41:19 +0200 Subject: [PATCH 360/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index e2982d2f4..29fbc0f2d 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit e2982d2f4c07d0252d59897d9002235f6c0a7f6c +Subproject commit 29fbc0f2da0c8d839898eb90b2d3a26c72ec85c8 From 959c4ce33775ecf588e4ad5794d1c340752510a5 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:33:25 +0200 Subject: [PATCH 361/459] updated .flake8 --- .flake8 | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index e31fb949f..de51ee708 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,32 @@ +# +# Author: Hari Sekhon +# Date: Mon Oct 21 15:57:10 2019 +0100 +# +# vim:ts=4:sts=4:sw=4:et +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# F l a k e 8 C o n f i g +# ============================================================================ # + +# https://flake8.pycqa.org/en/latest/user/configuration.html + [flake8] -ignore = E265,E402,F401 + max-line-length = 120 + +ignore = E265, + E402, + F401 + exclude = test*/* + max-complexity = 10 From 06d5e9469a49cf33c0c618ac5e05fbd4343dd4c8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 362/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index f32bb01be..057b80266 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit f32bb01be9e59606bf4a292c0a3a7e2ef1ac4540 +Subproject commit 057b80266f1bb0f3e36af2ae7f8fa228afc09074 From 49075c27e13a166fc5ac4dfd5582dfc9c936b928 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 363/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 29fbc0f2d..fdb270866 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 29fbc0f2da0c8d839898eb90b2d3a26c72ec85c8 +Subproject commit fdb2708661b5a58fdba4b7ba514246aa929e6c02 From cd4e61aa6c5d4ff4181342e5d8f8581a1b135493 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 2 Jul 2024 17:34:31 +0200 Subject: [PATCH 364/459] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 5174a5073..d94994282 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 5174a5073dbe25c8e5b1ac603b7b4f61b6ac7623 +Subproject commit d94994282274abf4279444fd85c0674a382e6696 From ec92f71bad8c41778516345018fd6295861aa67f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:17:37 +0300 Subject: [PATCH 365/459] added .envrc-python --- .envrc-python | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .envrc-python diff --git a/.envrc-python b/.envrc-python new file mode 100644 index 000000000..6bc2d65b1 --- /dev/null +++ b/.envrc-python @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y t h o n D i r E n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + break + fi +done + +# read .env too +#dotenv From bae58f9cbadd09648d416c363f73e5ae1d5dd582 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:19:10 +0300 Subject: [PATCH 366/459] updated .envrc --- .envrc | 145 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 18 deletions(-) diff --git a/.envrc b/.envrc index 51de39abb..7e481411c 100644 --- a/.envrc +++ b/.envrc @@ -14,41 +14,150 @@ # # ============================================================================ # -# P y t h o n D i r E n v +# D i r E n v # ============================================================================ # -# .envrc to auto-load the virtualenv inside the 'venv' directory if present - # https://direnv.net/man/direnv-stdlib.1.html -# See more .envrc files in: -# -# https://github.com/HariSekhon/DevOps-Bash-tools +# See Also: # # .envrc-aws # .envrc-gcp # .envrc-kubernetes +# direnv stdlib - loads .envrc from parent dir up to / +# +# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. +# +# bypasses security authorization though - use with care +#source_up +# +# source_up must be loaded before set -u otherwise gets this error: +# +# direnv: loading .envrc +# /bin/bash: line 226: $1: unbound variable +# +# source_up causes this error is up .envrc is found in parent directories: +# +# direnv: No ancestor .envrc found + set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +src="$(readlink -f "${BASH_SOURCE[0]}")" +srcdir="$(cd "$(dirname "$src")" && pwd)" -export COMPOSE_PROJECT_NAME="pytools" +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # -# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default -for venv in "$PWD/venv" "$HOME/venv"; do - if [ -f "$venv/bin/activate" ]; then - echo - echo "Virtualenv directory found in: $venv" - echo - echo "Activating Virtualenv inside the directory: $venv" +# Automatically install Pre-Commit Git hooks if not already present - # shellcheck disable=SC1091 - source "$venv/bin/activate" +if [ -f .pre-commit-config.yaml ]; then + if [ -d .git ]; then + if ! [ -f .git/hooks/pre-commit ]; then + pre-commit install + fi + fi +fi + +# ============================================================================ # +# D o c k e r C o m p o s e +# ============================================================================ # + +export COMPOSE_PROJECT_NAME="DevOps-Python-tools" + +# ============================================================================ # +# G i t H u b +# ============================================================================ # + +#export GITHUB_ORGANIZATION=HariSekhon + +# ============================================================================ # +# A n s i b l e +# ============================================================================ # + +# use the local repo's ansible.cfg rather than: +# +# $PWD/ansible.cfg +# ~/.ansible.cfg +# /etc/ansible/ansible.cfg +# +# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used +#export ANSIBLE_CONFIG="/path/to/ansible.cfg" + +# ============================================================================ # +# C l o u d f l a r e +# ============================================================================ # + +#export CLOUDFLARE_EMAIL=hari@... +#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens +#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module + +# export the variables for terraform +#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" +#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above + +# ============================================================================ # +# P y t h o n , A W S , G C P , T e r r a f o r m +# ============================================================================ # + +# XXX: safer to bring all these external .envrc inline if you're worried about changes +# to it bypassing 'direnv allow' authorization +load_if_exists(){ + # first arg is a path to a .envrc + # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes + # to pass the context name 'docker-desktop' to switch to + local envrc="$1" + shift + if ! [[ "$envrc" =~ ^/ ]]; then + envrc="$srcdir/$envrc" + fi + if [ -f "$envrc" ]; then + # prevent looping on symlinks to this .envrc if given + if [ "$(readlink "$envrc")" = "$src" ]; then + return + fi echo - break + echo "Loading $envrc" + # shellcheck disable=SC1090,SC1091 + . "$envrc" "$@" fi +} + +#load_if_exists ~/.envrc + + #.envrc-aws \ + #.envrc-gcp \ + #.envrc-terraform \ +# shellcheck disable=SC2043 +for envrc in \ + .envrc-python \ + ; do + load_if_exists "$envrc" done +if [[ "$PWD" =~ /aws/ ]]; then + load_if_exists .envrc-aws +fi + +if [[ "$PWD" =~ /gcp/ ]]; then + load_if_exists .envrc-gcp +fi + +if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then + load_if_exists .envrc-terraform +fi + +# ============================================================================ # +# K u b e r n e t e s +# ============================================================================ # + +if [ -f "$srcdir/.envrc-kubernetes" ]; then + load_if_exists .envrc-kubernetes docker-desktop +fi + +# ============================================================================ # + +echo # read .env too #dotenv From 13b3cde4939345bbb49086f16b627056958b2329 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:51:24 +0300 Subject: [PATCH 367/459] added .pre-commit-config.yaml --- .pre-commit-config.yaml | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..f3c5b1608 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,72 @@ +# +# Author: Hari Sekhon +# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024) +# +# vim:ts=2:sts=2:sw=2:et +# +# https///github.com/HariSekhon/Templates +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +--- +fail_fast: false +#exclude: *.tmp$ + +repos: + + # will accept anything that 'git clone' understands + # this means you can set this to a local git repo to develop your own hook repos interactively + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + # Common errors + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: forbid-new-submodules + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + + - repo: https://github.com/psf/black + rev: 24.8.0 + hooks: + - id: black + + # Git secrets Leaks + - repo: https://github.com/awslabs/git-secrets.git + # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error: + # + # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file + # + rev: 5357e18 + hooks: + - id: git-secrets + + - repo: https://github.com/markdownlint/markdownlint + rev: v0.12.0 + hooks: + - id: markdownlint + name: Markdownlint + description: Run markdownlint on your Markdown files + entry: mdl + language: ruby + files: \.(md|mdown|markdown)$ From 5c145c7d2e6459c61a9a55c0f6f7047e009ccbda Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 01:57:50 +0300 Subject: [PATCH 368/459] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3c5b1608..2b154867f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ # # vim:ts=2:sts=2:sw=2:et # -# https///github.com/HariSekhon/Templates +# https///github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # From c8c6623a14f7482ab9b222f63074f0bcbb55e657 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Aug 2024 20:33:29 +0300 Subject: [PATCH 369/459] updated .envrc --- .envrc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.envrc b/.envrc index 7e481411c..da56f607a 100644 --- a/.envrc +++ b/.envrc @@ -52,11 +52,11 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present -if [ -f .pre-commit-config.yaml ]; then - if [ -d .git ]; then - if ! [ -f .git/hooks/pre-commit ]; then - pre-commit install - fi +if [ -f .pre-commit-config.yaml ] && + [ -d .git ] && + type -P pre-commit &>/dev/null; then + if ! [ -f .git/hooks/pre-commit ]; then + pre-commit install fi fi From 9a5c385d480ecf1e05aacf4e0e753af684f9e262 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 00:12:38 +0300 Subject: [PATCH 370/459] updated .editorconfig --- .editorconfig | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 73c19b202..462f53206 100644 --- a/.editorconfig +++ b/.editorconfig @@ -40,7 +40,14 @@ end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true -[*.md] +[{*.md,*.hcl,*.tf,*.tfvars}] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.yml,*.yaml] indent_size = 2 indent_style = space end_of_line = lf From f410868fe774c96117e790f2bb7f6f2bd3fd9af6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 01:49:04 +0300 Subject: [PATCH 371/459] updated .envrc --- .envrc | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index da56f607a..920b4e16d 100644 --- a/.envrc +++ b/.envrc @@ -98,7 +98,7 @@ export COMPOSE_PROJECT_NAME="DevOps-Python-tools" #export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above # ============================================================================ # -# P y t h o n , A W S , G C P , T e r r a f o r m +# Load External Envrc Files If Present # ============================================================================ # # XXX: safer to bring all these external .envrc inline if you're worried about changes @@ -124,8 +124,15 @@ load_if_exists(){ fi } +# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc +# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses +# to install to system Python) #load_if_exists ~/.envrc +# ============================================================================ # +# P y t h o n +# ============================================================================ # + #.envrc-aws \ #.envrc-gcp \ #.envrc-terraform \ @@ -136,14 +143,26 @@ for envrc in \ load_if_exists "$envrc" done +# ============================================================================ # +# A W S +# ============================================================================ # + if [[ "$PWD" =~ /aws/ ]]; then load_if_exists .envrc-aws fi +# ============================================================================ # +# G C P +# ============================================================================ # + if [[ "$PWD" =~ /gcp/ ]]; then load_if_exists .envrc-gcp fi +# ============================================================================ # +# T e r r a f o r m +# ============================================================================ # + if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then load_if_exists .envrc-terraform fi From 283a9a3ca15ea2293c9ae2ed08a9ac77aa7d0bfa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 10 Aug 2024 12:06:01 +0300 Subject: [PATCH 372/459] updated .envrc --- .envrc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.envrc b/.envrc index 920b4e16d..15bc469ee 100644 --- a/.envrc +++ b/.envrc @@ -52,10 +52,23 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present +if ! type -P pre-commit &>/dev/null && + uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now..." + echo + brew install pre-commit + echo +fi + if [ -f .pre-commit-config.yaml ] && - [ -d .git ] && - type -P pre-commit &>/dev/null; then - if ! [ -f .git/hooks/pre-commit ]; then + type -P pre-commit &>/dev/null && + git rev-parse --is-inside-work-tree &>/dev/null; then + if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + echo + echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." + echo pre-commit install fi fi From f09fdd3279da8103197fc4be455be86eea23e2df Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 14 Aug 2024 23:17:32 +0200 Subject: [PATCH 373/459] updated .envrc --- .envrc | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.envrc b/.envrc index 15bc469ee..7555ecba9 100644 --- a/.envrc +++ b/.envrc @@ -52,14 +52,20 @@ srcdir="$(cd "$(dirname "$src")" && pwd)" # Automatically install Pre-Commit Git hooks if not already present -if ! type -P pre-commit &>/dev/null && - uname -s | grep -q Darwin && - type -P brew &>/dev/null; then - echo - echo "Pre-commit is not installed - installing now..." - echo - brew install pre-commit - echo +if ! type -P pre-commit &>/dev/null; then + if uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Homebrew..." + echo + brew install pre-commit + echo + elif type -P pip &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Pip..." + echo + pip install pre-commit + fi fi if [ -f .pre-commit-config.yaml ] && @@ -188,6 +194,8 @@ if [ -f "$srcdir/.envrc-kubernetes" ]; then load_if_exists .envrc-kubernetes docker-desktop fi +# ============================================================================ # +# . E n v # ============================================================================ # echo From d31b9e28d672eedc9624cfc2890daf982fd6acd9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 15 Aug 2024 16:35:10 +0200 Subject: [PATCH 374/459] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 503347f5d..cc918aa8d 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,11 @@ [![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) [![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) -[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) + [![ShellCheck](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml) [![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) From f3c5690d32c077c52b108ca1f83f88fbae06cfbe Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 22 Aug 2024 02:22:09 +0200 Subject: [PATCH 375/459] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b154867f..df14fb7d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,5 +68,6 @@ repos: name: Markdownlint description: Run markdownlint on your Markdown files entry: mdl + args: [-s, .markdownlint.rb] language: ruby files: \.(md|mdown|markdown)$ From 4546c7688be6a4e0a12b333b8fa123c0aae7a91c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 22 Aug 2024 02:22:10 +0200 Subject: [PATCH 376/459] added .markdownlint.rb --- .markdownlint.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .markdownlint.rb diff --git a/.markdownlint.rb b/.markdownlint.rb new file mode 100644 index 000000000..e69de29bb From db58f5309bcf6fe5d145bb65afdc501b0c1494d6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:18 +0200 Subject: [PATCH 377/459] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df14fb7d9..9e9c1c291 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,6 +68,6 @@ repos: name: Markdownlint description: Run markdownlint on your Markdown files entry: mdl - args: [-s, .markdownlint.rb] + args: [-s, .mdl.rb] language: ruby files: \.(md|mdown|markdown)$ From 0839b2b170aa75b94c39263577660b3cac5e25ee Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:19 +0200 Subject: [PATCH 378/459] added .mdlrc --- .mdlrc | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .mdlrc diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 000000000..27e5b6895 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,5 @@ +mdlrc_dir = File.expand_path('..', __FILE__) + +style_file = File.join(mdlrc_dir, '.mdl.rb') + +style style_file From c8ec774ef167b959685256791528eba64dbd8b32 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:35:19 +0200 Subject: [PATCH 379/459] added .mdl.rb --- .mdl.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .mdl.rb diff --git a/.mdl.rb b/.mdl.rb new file mode 100644 index 000000000..db6a3373f --- /dev/null +++ b/.mdl.rb @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# vim:ts=4:sts=4:sw=4:et:filetype=ruby +# +# Author: Hari Sekhon +# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +all +#exclude_rule 'MD001' +#exclude_rule 'MD003' +#exclude_rule 'MD005' +exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af +#exclude_rule 'MD012' +exclude_rule 'MD013' # long lines cannot be split if they are URLs +#exclude_rule 'MD022' +#exclude_rule 'MD025' +#exclude_rule 'MD031' +#exclude_rule 'MD032' +exclude_rule 'MD033' # inline HTML is important for formatting +#exclude_rule 'MD039' +#exclude_rule 'MD056' From cfa61d4d041dba2306dfdfbf568226209d656f56 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Sep 2024 23:36:37 +0200 Subject: [PATCH 380/459] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index ab20cfa8d..0b52378ff 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -52,7 +52,7 @@ retry(){ if [ "$(uname -s)" = Darwin ]; then echo "Bootstrapping Mac" # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make) - #retry "$srcdir/install_homebrew.sh" + #retry "$srcdir/../install/install_homebrew.sh" if command -v brew 2>&1; then # fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do From 63bde7fef5cf084fa7fe4a0a212d60d275f46ef2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 381/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 057b80266..e7c73c3c4 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 057b80266f1bb0f3e36af2ae7f8fa228afc09074 +Subproject commit e7c73c3c436c41901d1b51c872e5c3e2b22e2b07 From 16a294f584f532cde95b2df7401cc50234bdd3d8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 382/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index fdb270866..092819070 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit fdb2708661b5a58fdba4b7ba514246aa929e6c02 +Subproject commit 092819070eda268f0a4c6f8b373290c09a9de421 From c91a96c1ebd6a698d568d6a36d758872da9804b4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:19 +0200 Subject: [PATCH 383/459] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 8597f692e..ace6d6400 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 8597f692e4e4abd59b58fb24c7fa79449a75c8c0 +Subproject commit ace6d6400c1f647f2e96bd6dc9b730cf1232847a From b53766eb78df127afdcc81e3dde0a50a8f9dabe8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 16:05:20 +0200 Subject: [PATCH 384/459] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index d94994282..b8d565217 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit d94994282274abf4279444fd85c0674a382e6696 +Subproject commit b8d565217a111514a2e061cec896f258712985fb From 82e9390f13636322f39891aa5cc372a4496428c7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 17:15:58 +0200 Subject: [PATCH 385/459] updated README --- README.md | 196 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 116 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index cc918aa8d..4fd1177dc 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) [![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) -[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/HariSekhon/) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) -- [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... +### Knowledge -- [The Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) -- [Nagios Plugin Kafka](https://github.com/HariSekhon/Nagios-Plugin-Kafka) - Kafka API pub/sub Nagios Plugin written in Scala with Kerberos support + -- [Diagrams-as-Code](https://github.com/HariSekhon/Diagrams-as-Code) - Cloud & Open Source architecture diagrams with Python & D2 source code provided - automatically regenerated via GitHub Actions CI/CD - AWS, GCP, Kubernetes, Jenkins, ArgoCD, Traefik, Kong API Gateway, Nginx, Redis, PostgreSQL, Kafka, Spark, web farms, event processing... +### DevOps Code -- [Knowledge-Base](https://github.com/HariSekhon/Knowledge-Base) - IT Knowledge Base from 20 years in DevOps, Linux, Cloud, Big Data, AWS, GCP etc. +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Python-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Perl-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Perl-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Golang-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Golang-tools) -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +### Containerization -- https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) -### Stargazers over time +### CI/CD -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) -[git.io/python-tools](https://git.io/python-tools) +### DBA - SQL -[git.io/pytools](https://git.io/pytools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) + +### DevOps Reloaded + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) + +### Misc + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) + +The rest of my original source repos are +[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). + +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). + + +![](https://hit.yhype.me/github/profile?user_id=2211051) + + From e27764e4a9ed13170d3585259ea6267be3b7f62d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 4 Sep 2024 17:17:33 +0200 Subject: [PATCH 386/459] updated README --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4fd1177dc..5fa8938af 100644 --- a/README.md +++ b/README.md @@ -534,14 +534,17 @@ Does nothing: [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) -### Misc +### Templates +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) + +### Misc + [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) From 6259d5a9e744abde66b0ff91c15b194e8104b075 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:19:58 +0200 Subject: [PATCH 387/459] updated .mdl.rb --- .mdl.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/.mdl.rb b/.mdl.rb index db6a3373f..f8f9b004c 100644 --- a/.mdl.rb +++ b/.mdl.rb @@ -25,5 +25,6 @@ #exclude_rule 'MD031' #exclude_rule 'MD032' exclude_rule 'MD033' # inline HTML is important for formatting +exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines #exclude_rule 'MD039' #exclude_rule 'MD056' From e4eac61b95c830eb6ee80027214d72e9be9dcd6e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:30:55 +0200 Subject: [PATCH 388/459] updated README --- README.md | 176 +++++++++++++++++++++++++++--------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 5fa8938af..955eb8cb6 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Cloud & Big Data Contractor, United Kingdom [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/)
*(you're welcome to connect with me on LinkedIn)* -**Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries** +**Make sure you run `make update` if updating and not just `git pull` as you will often need the latest library submodule and possibly new upstream libraries** ## Quick Start @@ -163,14 +163,14 @@ Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://git ### Usage -All programs come with a ```--help``` switch which includes a program description and the list of command line options. +All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. ### DevOps Python Tools - Inventory - Linux: - - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -179,115 +179,115 @@ Environment variables are supported for convenience and also to hide credentials - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` - - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - ```find_active_server.py``` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` + - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [Amazon Web Services](https://aws.amazon.com/): - - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all docker images eg. `docker_pull_all_images_all_tags.sh` + - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` - [Spark](https://spark.apache.org/) & Data Format Converters: - - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter - - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas - - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```xml_to_json.py``` - XML to JSON converter - - ```json_to_xml.py``` - JSON to XML converter - - ```json_to_yaml.py``` - JSON to YAML converter - - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - - ```yaml_to_json.py``` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) - - see also ```validate_*.py``` further down for all these formats and more + - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter + - `spark_parquet_to_avro.py` - PySpark Parquet => Avro converter + - `spark_csv_to_avro.py` - PySpark CSV => Avro converter, supports both inferred and explicit schemas + - `spark_csv_to_parquet.py` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas + - `spark_json_to_avro.py` - PySpark JSON => Avro converter + - `spark_json_to_parquet.py` - PySpark JSON => Parquet converter + - `xml_to_json.py` - XML to JSON converter + - `json_to_xml.py` - JSON to XML converter + - `json_to_yaml.py` - JSON to YAML converter + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) + - see also `validate_*.py` further down for all these formats and more - [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: - [Ambari](https://hortonworks.com/apache/ambari/): - - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API + - `ambari_blueprints.py` - Blueprint cluster templating and deployment tool using Ambari API - list blueprints - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool - - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API - - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable using this tool + - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts + - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API + - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - ```hdfs_find_replication_factor_1.py``` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - ```hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - ```hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output The following programs can all optionally filter by database / table name regex: - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables - - ```hive_tables_row_column_counts.py``` / ```impala_tables_row_column_counts.py``` - outputs tables row and column counts. Useful for finding unusually big tables - - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_metadata.py``` / ```impala_tables_metadata.py``` - outputs for each table the matching regex metadata DDL property from describe table - - ```hive_tables_locations.py``` / ```impala_tables_locations.py``` - outputs for each table its data location + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL property from describe table + - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables - - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop -- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig_udfs.jy` - Pig Jython UDFs for Hadoop +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required - - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node + - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA + - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA + - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node + - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator + - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server + - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server + - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node + - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- ```selenium_hub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - - ```validate_*.py``` - validate files, directory trees and/or standard input streams + - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: - Avro - CSV @@ -326,7 +326,7 @@ Download the DevOps Python Tools and Pylib git repos as zip files: -Unzip both and move Pylib to the ```pylib``` folder under DevOps Python Tools. +Unzip both and move Pylib to the `pylib` folder under DevOps Python Tools. ```shell unzip devops-python-tools-master.zip @@ -450,7 +450,7 @@ make update This will git pull and then git submodule update which is necessary to pick up corresponding library updates. -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run ```make update-no-recompile``` (will miss new library dependencies - do full ```make update``` if you encounter issues). +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter issues). ### Testing @@ -476,9 +476,9 @@ You might also be interested in the following really nice Jupyter notebook for H -### Stargazers over time +## Star History -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) +[![Star History Chart](https://api.star-history.com/svg?repos=HariSekhon/DevOps-Python-tools&type=Date)](https://star-history.com/#HariSekhon/DevOps-Python-tools&Date) [git.io/python-tools](https://git.io/python-tools) From 73ff2ef8fe71271639bea252a671c6a70fbf5830 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 19:32:22 +0200 Subject: [PATCH 389/459] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 955eb8cb6..8c3ad36f1 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,7 @@ Environment variables are supported for convenience and also to hide credentials - [Hadoop](http://hadoop.apache.org/) HDFS: - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing `hdfs dfs -cat | md5sum`) - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output From 94d58f49cb50bbe8ef645d3a785914cd6231b6ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 16 Sep 2024 20:48:32 +0200 Subject: [PATCH 390/459] updated README.md --- README.md | 253 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 181 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 8c3ad36f1..f9fdde42d 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,8 @@ cd pytools make ``` -To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension instead of `.py`: +To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension +instead of `.py`: ```shell make anonymize.pyc @@ -165,12 +166,15 @@ Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://git All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list +eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and +often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. ### DevOps Python Tools - Inventory - Linux: - - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing + - lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -181,27 +185,57 @@ Environment variables are supported for convenience and also to hide credentials - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) - - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes + communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support + teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel + before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation + important for anonymization rules, as well as maintaining file content order. On servers this parallelization can + result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file + basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, + useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). + Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See + further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in + your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [Amazon Web Services](https://aws.amazon.com/): - - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally + filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have + never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. + Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for + users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to + [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is + triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and + [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP + address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its + accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via + VPC connector routing + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with + their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` - - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all docker images eg. `docker_pull_all_images_all_tags.sh` + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a + docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very + useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only + the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests + across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official + `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned + to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy + shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. + `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all + docker images eg. `docker_pull_all_images_all_tags.sh` - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` - [Spark](https://spark.apache.org/) & Data Format Converters: - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter @@ -213,7 +247,13 @@ Environment variables are supported for convenience and also to hide credentials - `xml_to_json.py` - XML to JSON converter - `json_to_xml.py` - JSON to XML converter - `json_to_yaml.py` - JSON to YAML converter - - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for + pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and + [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / + directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard + output for convenient command line chaining and redirection, optionally continues on error, collects broken records + to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not + technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) - see also `validate_*.py` further down for all these formats and more - [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: @@ -223,69 +263,119 @@ Environment variables are supported for convenience and also to hide credentials - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control + - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for + proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable using this tool + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable + using this tool - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing `hdfs dfs -cat | md5sum`) - - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to + replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file + or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data + layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster + than doing `hdfs dfs -cat | md5sum`) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree + showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format + to standard output The following programs can all optionally filter by database / table name regex: - - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala table - - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation between cluster migrations - - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for finding unusually wide tables - - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. Useful for finding unusually big tables - - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL property from describe table + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala + table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation + between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for + finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. + Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where + any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or + subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. + Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain + NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL + property from describe table - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting tables - - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, + with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI + tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting + tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, + giving per region row counts and % of total rows for the table as well as median and quartile row counts per + regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in + an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row + counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating + on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an + optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or + memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average + since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / + total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or + average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, + write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular + RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received + the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew + and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase + tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - `pig_udfs.jy` - Pig Jython UDFs for Hadoop -- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, + useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex + content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for + which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you + want to query cluster wide information available from any online peer (eg. Elasticsearch) + - The following are simplified specialisations of the above program, just pass host arguments, all the details have + been baked in, no switches required - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run + multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server + (run multiple of these for load balancing) - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, + Catalog or Statestore - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) + - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the + [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - + useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks + session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot + debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as + Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: @@ -297,14 +387,20 @@ Environment variables are supported for convenience and also to hide credentials - Parquet - XML - YAML - - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) - - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies + - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, + `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) + - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for + things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images + [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & + Linux technologies ### Detailed Build Instructions #### Python VirtualEnv localized installs -The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages instead of installing from PyPI then follow the Manual Build section below. +The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it +detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages +instead of installing from PyPI then follow the Manual Build section below. ### Manual Setup @@ -337,7 +433,8 @@ mv -v pylib-master pylib mv -vf pylib pytools/ ``` -Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). +Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal +mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files. @@ -355,9 +452,11 @@ sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt #### Mac OS X -The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing `git` is enough to trigger Xcode install). +The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing +`git` is enough to trigger Xcode install). -I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): +I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for +development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): ```shell bash-tools/install/install_homebrew.sh @@ -373,7 +472,8 @@ If failing to build an OpenSSL lib dependency, just prefix the build command lik sudo OPENSSL_INCLUDE=/usr/local/opt/openssl/include OPENSSL_LIB=/usr/local/opt/openssl/lib ... ``` -You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is caused by pip 10 vs pip 9 and downgrading will work around it: +You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is +caused by pip 10 vs pip 9 and downgrading will work around it: ```shell sudo pip install --upgrade pip==9.0.1 @@ -398,7 +498,8 @@ Run like so: jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` -The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. +The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop +APIs. See below for procedure to install Jython if you don't already have it. @@ -422,7 +523,10 @@ Then add the Jython install bin directory to the $PATH or specify the full path ### Configuration for Strict Domain / FQDN validation -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my +[PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, +`.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal +environments). ### Python SSL certificate verification problems @@ -433,7 +537,8 @@ If you end up with an error like: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:765) ``` -It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One quick fix is to do the following: +It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One +quick fix is to do the following: ```shell sudo pip uninstall -y certifi && @@ -450,7 +555,9 @@ make update This will git pull and then git submodule update which is necessary to pick up corresponding library updates. -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter issues). +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each +time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter +issues). ### Testing @@ -466,13 +573,15 @@ To trigger all tests run: make test ``` -which will start with the underlying libraries, then move on to top level integration tests and functional tests using docker containers if docker is available. +which will start with the underlying libraries, then move on to top level integration tests and functional tests using +docker containers if docker is available. ### Contributions Patches, improvements and even general feedback are welcome in the form of GitHub pull requests and issue tickets. -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another +Hortonworks guy Jonas Straub: From 68cdf7de64f0e0e6193c99ad330dcd4e92e6b4f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 19 Sep 2024 01:17:17 +0200 Subject: [PATCH 391/459] removed .markdownlint.rb --- .markdownlint.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .markdownlint.rb diff --git a/.markdownlint.rb b/.markdownlint.rb deleted file mode 100644 index e69de29bb..000000000 From 2ff0c6b46c0cb830261565739e20e365dad717e3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 18:40:41 +0100 Subject: [PATCH 392/459] updated azure-pipelines.yml --- azure-pipelines.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e26e7f760..b430c58dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,19 +22,26 @@ trigger: - master +variables: + # ubuntu version + os_version: '22.04' + pool: # there is no /dev/stderr on this azure build! #vmImage: 'ubuntu-latest' - # Ubuntu 16.04 required for docker container support, looks like 18.04 works too - vmImage: 'ubuntu-22.04' + #vmImage: 'ubuntu-22.04' + vmImage: 'ubuntu-$(os_version)' # unprivileged container without sudo, cannot install dependencies -#container: ubuntu:18.04 +#container: ubuntu:22.04 steps: + - script: cat /etc/*-release + displayName: OS Release + # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' - script: env | sort - displayName: env + displayName: Environment # doesn't work in container due to unprivileged execution and lack of sudo #- script: sudo apt-get update && sudo apt-get install -y git make @@ -50,7 +57,7 @@ steps: # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( - script: | - sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c ' + sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c ' set -ex cd /code setup/ci_bootstrap.sh From def50a98abb78cca6c938bb9fcca1ac660fde7a8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 393/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index e7c73c3c4..399bc52b2 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit e7c73c3c436c41901d1b51c872e5c3e2b22e2b07 +Subproject commit 399bc52b2b9fa0d3aa4b60d2eb18cb1e3b3ae0b6 From 38c02005e8cd21b410901588698693ea6f26eebf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 394/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 092819070..81714c62c 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 092819070eda268f0a4c6f8b373290c09a9de421 +Subproject commit 81714c62ccfee06feef0ab8cef5c1485bff7d9c0 From 566d308bd84fa3a7ff327d7f05f67a83bb05c0ea Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 395/459] updated submodule sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index ace6d6400..575080b54 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit ace6d6400c1f647f2e96bd6dc9b730cf1232847a +Subproject commit 575080b548fec8b2420520e161973a861cbbce30 From fbca030c08ef4b3af3a131252dfbef8459bdd031 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 21 Sep 2024 22:21:04 +0100 Subject: [PATCH 396/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index b8d565217..da9d4c1ab 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit b8d565217a111514a2e061cec896f258712985fb +Subproject commit da9d4c1ab5787bbd3802db78a2330ad713ee3cbe From b77d999a26ca7f335eccc397881d3981536d8dc0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:26 +0100 Subject: [PATCH 397/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 399bc52b2..4c661ca88 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 399bc52b2b9fa0d3aa4b60d2eb18cb1e3b3ae0b6 +Subproject commit 4c661ca889cbef24e989f27c184d12d02cb99c98 From c6ab1cc4353d5a55bff3b41a775635537a9c5f7b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:26 +0100 Subject: [PATCH 398/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 81714c62c..666501ada 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 81714c62ccfee06feef0ab8cef5c1485bff7d9c0 +Subproject commit 666501adaf43473a0c94b2ace714598a80cdb549 From 1981197b571c7f661bc912648878f6ef46cf37ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 18:48:27 +0100 Subject: [PATCH 399/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index da9d4c1ab..6cee5eb76 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit da9d4c1ab5787bbd3802db78a2330ad713ee3cbe +Subproject commit 6cee5eb764acd9f9198bd83f9af3b17bf49e1283 From 3e40bd427b4780b74718a7dbdf395cf33484a814 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:12 +0100 Subject: [PATCH 400/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4c661ca88..3dd58df8c 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4c661ca889cbef24e989f27c184d12d02cb99c98 +Subproject commit 3dd58df8cbaeaf22661c2cf68f289535508c4685 From 5b7e885adc73fada6c37497d18fabc7c1290c47f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:13 +0100 Subject: [PATCH 401/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 666501ada..c90dd3e3a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 666501adaf43473a0c94b2ace714598a80cdb549 +Subproject commit c90dd3e3ae1ccb64ed0e668750fa594b3c709563 From bff7195cde897972a547360784d5a0f7129fd559 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 19:15:13 +0100 Subject: [PATCH 402/459] updated submodule templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 6cee5eb76..2d12574ce 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 6cee5eb764acd9f9198bd83f9af3b17bf49e1283 +Subproject commit 2d12574ceafbcb13f559dfa46693024616b8e619 From c1a5672830bd8eb9628f505a305186cbbd65100c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:34:26 +0100 Subject: [PATCH 403/459] updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f377f523f..80ff670d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ GitPython==2.1.15 #happybase==1.0.0 humanize==0.5.1 -impyla==0.16.0 +impyla==0.19.0 jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 From 5c900849a8d38f989a1343c3c36d54d7026adb03 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:38:31 +0100 Subject: [PATCH 404/459] updated submodule bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 3dd58df8c..adc18db20 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 3dd58df8cbaeaf22661c2cf68f289535508c4685 +Subproject commit adc18db20909106dbf4d6ededa1c6c66d5c28e8c From f373795d24110882a5e8a036484dfa0d5a423ded Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 20:38:31 +0100 Subject: [PATCH 405/459] updated submodule pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index c90dd3e3a..eb10979e3 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit c90dd3e3ae1ccb64ed0e668750fa594b3c709563 +Subproject commit eb10979e3572ef37b35bda3653b25d7a45e25169 From 2e739a43d79ff855a58c1d76725065c7d1257d34 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 22 Sep 2024 22:12:45 +0100 Subject: [PATCH 406/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index adc18db20..6fc486cfb 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit adc18db20909106dbf4d6ededa1c6c66d5c28e8c +Subproject commit 6fc486cfb85886c8b125c3649b80de1762eaba21 From 84e699a8bdae550158a4940234c98996666efbbd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Sep 2024 11:52:18 +0100 Subject: [PATCH 407/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 6fc486cfb..176f383c9 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 6fc486cfb85886c8b125c3649b80de1762eaba21 +Subproject commit 176f383c9148361e04dfde142054ce2efbdbb5e6 From b580060646cdba0f99af7e8bb557a53491d223a6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 23 Sep 2024 12:22:58 +0100 Subject: [PATCH 408/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 176f383c9..34f300de0 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 176f383c9148361e04dfde142054ce2efbdbb5e6 +Subproject commit 34f300de0e37f5cba5176796560c99e3780a7cc1 From c9f8ba3236600b8666053f3c9fc7bdbcfeff7dd2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 24 Sep 2024 00:07:20 +0100 Subject: [PATCH 409/459] updated .envrc --- .envrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.envrc b/.envrc index 7555ecba9..bafe8df0b 100644 --- a/.envrc +++ b/.envrc @@ -201,3 +201,5 @@ fi echo # read .env too #dotenv + +load_if_exists .envrc.local From 1b5dcec3b4c4ca4373615075b6db22c589a4d870 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 8 Oct 2024 09:27:46 +0300 Subject: [PATCH 410/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 34f300de0..4f4be21ed 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 34f300de0e37f5cba5176796560c99e3780a7cc1 +Subproject commit 4f4be21ed3006a756971134afa71c283388b111d From ee32cb23b8c4617015737f9c4938c852c0c03f19 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Oct 2024 18:35:23 +0300 Subject: [PATCH 411/459] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e9c1c291..f615a7725 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,14 +30,14 @@ repos: hooks: - id: check-yaml # Common errors - - id: end-of-file-fixer + #- id: end-of-file-fixer # ruins .gitignore Icon\r - id: trailing-whitespace args: [--markdown-linebreak-ext=md] # Git style - id: check-added-large-files - id: check-merge-conflict - id: check-vcs-permalinks - - id: forbid-new-submodules + #- id: forbid-new-submodules # Cross platform - id: check-case-conflict - id: mixed-line-ending From f04002186cf34e64c7a7503d91144fa7521df597 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 11 Oct 2024 17:51:20 +0300 Subject: [PATCH 412/459] added markdown.yaml --- .github/workflows/markdown.yaml | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/markdown.yaml diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml new file mode 100644 index 000000000..cb5395963 --- /dev/null +++ b/.github/workflows/markdown.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# M a r k D o w n +# ============================================================================ # + +--- +name: Markdown + +on: + push: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + Markdown: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Markdown + uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master From 0e7f2891dca92120dc47009e1e496a871b8ac3cc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 11 Oct 2024 18:00:17 +0300 Subject: [PATCH 413/459] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f9fdde42d..6d11affff 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ [![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) +[![Markdown](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) [![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) [![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) From ac683274c27147af43de231a903b2e6945bd63b3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:29:36 +0400 Subject: [PATCH 414/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 4f4be21ed..956f47624 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 4f4be21ed3006a756971134afa71c283388b111d +Subproject commit 956f47624a5fcefdf46c5d675298e0526121713c From b9a12fcc4a414016b44da2af0cd060f5b97dbe57 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:32:27 +0400 Subject: [PATCH 415/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 956f47624..cf080c2c5 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 956f47624a5fcefdf46c5d675298e0526121713c +Subproject commit cf080c2c584de729b3bbbcefb3cfa353cf64216a From 5206c0ff64ecd8f3b08d050859be1b6080011631 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:47:40 +0400 Subject: [PATCH 416/459] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f615a7725..6ae7e968c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,10 +46,10 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] - - repo: https://github.com/psf/black - rev: 24.8.0 - hooks: - - id: black + #- repo: https://github.com/psf/black + # rev: 24.8.0 + # hooks: + # - id: black # Git secrets Leaks - repo: https://github.com/awslabs/git-secrets.git From ea2ed7d241ea06422fea231a2fcc513b17597ab6 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:47:41 +0400 Subject: [PATCH 417/459] permitted space in section headers to allow for AWS [profile blah] sections --- validate_ini.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validate_ini.py b/validate_ini.py index 54a01cc63..b38b3ab6a 100755 --- a/validate_ini.py +++ b/validate_ini.py @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.12.2' +__version__ = '0.12.3' class IniValidatorTool(CLI): @@ -72,7 +72,7 @@ def __init__(self): self.re_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I) # In Windows ini key cannot contain equals sign = or semicolon ; # key=val or [section] - self.re_ini_section = re.compile(r'^\s*\[([\w=\:\.-]+)\]\s*$') + self.re_ini_section = re.compile(r'^\s*\[([\w\s=\:\.-]+)\]\s*$') self.re_ini_key = re.compile(r'^\s*(?:[^\[;=]+)s*$') # INI value can be anything .* so not regex'ing it self.valid_ini_msg = ' => INI OK' From 67c680b35148ee03a249575ed54a0c047ead4144 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 22 Nov 2024 17:52:07 +0400 Subject: [PATCH 418/459] updated .pre-commit-config.yaml --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ae7e968c..787fb79ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,6 +46,7 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] + # rewrites python files with useless changes like changing single quotes to double quotes #- repo: https://github.com/psf/black # rev: 24.8.0 # hooks: From 510fd69c3f7c7ee0a98ad81ae26123c8f7fb1be9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 24 Nov 2024 04:06:10 +0400 Subject: [PATCH 419/459] updated test_dockerhub_search.sh --- tests/test_dockerhub_search.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index 92e3d2e9d..2a4f9c48e 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -36,7 +36,7 @@ check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED # shellcheck disable=SC2016 -check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" +check '[ "$(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep -c "^harisekhon/[A-Za-z0-9_-]*$")" = 40 ]' "DockerHub Search quiet mode for shell scripting" echo echo From 22fa4cff9849d3230cbbb48a1733c6403286510b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 8 Dec 2024 00:03:50 +0700 Subject: [PATCH 420/459] updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09026fc6a..ab6c6499d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,7 +37,7 @@ workflows: jobs: build: docker: - - image: cimg/base:2021.04 + - image: cimg/base:2024.12 resource_class: small steps: - checkout From 6b56bd27261b489177dece866bc49ca7fdd2364a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 2 Feb 2025 00:21:59 +0700 Subject: [PATCH 421/459] added lint exception for on: truthy --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/checkov.yaml | 2 +- .github/workflows/codeowners.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_11.yaml | 2 +- .github/workflows/debian_12.yaml | 2 +- .github/workflows/docker_pytools_alpine.yaml | 2 +- .github/workflows/docker_pytools_debian.yaml | 2 +- .github/workflows/docker_pytools_fedora.yaml | 2 +- .github/workflows/docker_pytools_ubuntu.yaml | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/grype.yaml | 2 +- .github/workflows/json.yaml | 2 +- .github/workflows/kics.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_11.yaml | 2 +- .github/workflows/mac_12.yaml | 2 +- .github/workflows/markdown.yaml | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.11.yaml | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/shellcheck.yaml | 2 +- .github/workflows/trivy.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_22.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/validate.yaml | 2 +- .github/workflows/xml.yaml | 2 +- .github/workflows/yaml.yaml | 2 +- 38 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a678d6592..dc5dfa86c 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -14,7 +14,7 @@ --- name: Alpine -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index fd88e54cb..a522558aa 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -14,7 +14,7 @@ --- name: Alpine 3 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 8f31b2672..209b3cef1 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -22,7 +22,7 @@ --- name: Checkov -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 4dc3a09c3..6a0edd459 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -18,7 +18,7 @@ --- name: CodeOwners -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d56453263..7794a2914 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -14,7 +14,7 @@ --- name: Debian -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 735603e28..e8fec2ae1 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -14,7 +14,7 @@ --- name: Debian 10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml index de3b4b96f..3bebff0c9 100644 --- a/.github/workflows/debian_11.yaml +++ b/.github/workflows/debian_11.yaml @@ -14,7 +14,7 @@ --- name: Debian 11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml index d543710b1..3dbcdd303 100644 --- a/.github/workflows/debian_12.yaml +++ b/.github/workflows/debian_12.yaml @@ -14,7 +14,7 @@ --- name: Debian 12 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index 769f90e49..7119d6852 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Alpine) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index b2b718ac0..24f8f21dd 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Debian) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 70c545db6..ad1a5ed8d 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Fedora) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index e69018c31..884917919 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Ubuntu) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 0975be19d..3dfae11a4 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -14,7 +14,7 @@ --- name: Fedora -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index c7ff16fd4..3c1aecadf 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -20,7 +20,7 @@ --- name: Fork Sync -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 94398083f..bedec7659 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -22,7 +22,7 @@ --- name: Fork Update PR -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml index 6a321f86e..351180e94 100644 --- a/.github/workflows/grype.yaml +++ b/.github/workflows/grype.yaml @@ -18,7 +18,7 @@ --- name: Grype -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index f68e309e5..83fa5ef83 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -20,7 +20,7 @@ --- name: JSON -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index 033f4331e..f7f7383fe 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -18,7 +18,7 @@ --- name: Kics -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index ac6957e87..4959ef720 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -14,7 +14,7 @@ --- name: Mac -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml index 2d948e136..64ec035b6 100644 --- a/.github/workflows/mac_11.yaml +++ b/.github/workflows/mac_11.yaml @@ -14,7 +14,7 @@ --- name: Mac 11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml index 7d2bf2dc6..5fa32d10c 100644 --- a/.github/workflows/mac_12.yaml +++ b/.github/workflows/mac_12.yaml @@ -14,7 +14,7 @@ --- name: Mac 12 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml index cb5395963..72e301ba0 100644 --- a/.github/workflows/markdown.yaml +++ b/.github/workflows/markdown.yaml @@ -18,7 +18,7 @@ --- name: Markdown -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index de6f79440..7d953dfd7 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -14,7 +14,7 @@ --- name: Python 3.10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml index 8e2d041ac..91658c8e9 100644 --- a/.github/workflows/python3.11.yaml +++ b/.github/workflows/python3.11.yaml @@ -14,7 +14,7 @@ --- name: Python 3.11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 58ad6c210..04c7146c7 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -14,7 +14,7 @@ --- name: Python 3.7 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index f1c806dc2..37ad93654 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -14,7 +14,7 @@ --- name: Python 3.8 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 31c7d0ab8..9cf18e290 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -14,7 +14,7 @@ --- name: Python 3.9 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 43b3f5bd2..3ac4c40b9 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -20,7 +20,7 @@ --- name: Semgrep Cloud -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index f1e452f88..94fd43a22 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -22,7 +22,7 @@ --- name: Semgrep -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 753a6a495..ea4a967eb 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -20,7 +20,7 @@ --- name: ShellCheck -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 26f01fa94..c31ec991e 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -20,7 +20,7 @@ --- name: Trivy -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 0d7fd952c..8edea5c2d 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 5145177bb..ae5ec97ef 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu 20.04 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml index b9bf17a95..d5dfb94a8 100644 --- a/.github/workflows/ubuntu_22.04.yaml +++ b/.github/workflows/ubuntu_22.04.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu 22.04 -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 53307dd73..2efa3ac6d 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -14,7 +14,7 @@ --- name: GitHub Actions Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 94a8c007a..468d875e3 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -20,7 +20,7 @@ --- name: Validation -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml index 8b07ed256..3f6265a6a 100644 --- a/.github/workflows/xml.yaml +++ b/.github/workflows/xml.yaml @@ -20,7 +20,7 @@ --- name: XML -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index d252f67d2..5c3118c33 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -20,7 +20,7 @@ --- name: YAML -on: +on: # yamllint disable-line rule:truthy push: branches: - master From 7738761d337f64222906c6a623ed410dfb18fa44 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 2 Feb 2025 00:23:39 +0700 Subject: [PATCH 422/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index cf080c2c5..a97248860 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit cf080c2c584de729b3bbbcefb3cfa353cf64216a +Subproject commit a97248860172c6bb3534d08ad38c4cfdfd09c306 From 406fce6a86160038a5ca122f54d0a401fab1e93b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Feb 2025 20:35:59 +0700 Subject: [PATCH 423/459] updated README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d11affff..22741ad41 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) +[![LineCount](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2)](https://github.com/boyter/scc/) +[![Cocomo](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2&category=cocomo)](https://github.com/boyter/scc/) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) [![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) From c9f6bce6e1972482bd39a67a793c6c241398933a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2025 14:00:14 +0700 Subject: [PATCH 424/459] changed concurrency lock to be workflow-ref format --- .github/workflows/alpine.yaml | 2 +- .github/workflows/alpine_3.yaml | 2 +- .github/workflows/centos.yaml.disabled | 2 +- .github/workflows/centos7.yaml.disabled | 2 +- .github/workflows/centos8.yaml.disabled | 2 +- .github/workflows/codeowners.yaml | 2 +- .github/workflows/debian.yaml | 2 +- .github/workflows/debian_10.yaml | 2 +- .github/workflows/debian_11.yaml | 2 +- .github/workflows/debian_12.yaml | 2 +- .github/workflows/debian_6.yaml.disabled | 2 +- .github/workflows/debian_7.yaml.disabled | 2 +- .github/workflows/debian_8.yaml.disabled | 2 +- .github/workflows/debian_9.yaml.disabled | 2 +- .github/workflows/fedora.yaml | 2 +- .github/workflows/fork-sync.yaml | 2 +- .github/workflows/fork-update-pr.yaml | 2 +- .github/workflows/grype.yaml | 2 +- .github/workflows/kics.yaml | 2 +- .github/workflows/mac.yaml | 2 +- .github/workflows/mac_11.yaml | 2 +- .github/workflows/mac_12.yaml | 2 +- .github/workflows/pypy2.yaml.disabled | 2 +- .github/workflows/pypy3.yaml.disabled | 2 +- .github/workflows/python2.7.yaml.disabled | 2 +- .github/workflows/python3.10.yaml | 2 +- .github/workflows/python3.11.yaml | 2 +- .github/workflows/python3.6.yaml.disabled | 2 +- .github/workflows/python3.7.yaml | 2 +- .github/workflows/python3.8.yaml | 2 +- .github/workflows/python3.9.yaml | 2 +- .github/workflows/semgrep-cloud.yaml | 2 +- .github/workflows/semgrep.yaml | 2 +- .github/workflows/shellcheck.yaml | 2 +- .github/workflows/trivy.yaml | 2 +- .github/workflows/ubuntu.yaml | 2 +- .github/workflows/ubuntu_14.04.yaml.disabled | 2 +- .github/workflows/ubuntu_16.04.yaml.disabled | 2 +- .github/workflows/ubuntu_18.04.yaml.disabled | 2 +- .github/workflows/ubuntu_20.04.yaml | 2 +- .github/workflows/ubuntu_22.04.yaml | 2 +- .github/workflows/ubuntu_github.yaml | 2 +- .github/workflows/validate.yaml | 2 +- .github/workflows/yaml.yaml | 2 +- 44 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index dc5dfa86c..9bed93efb 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index a522558aa..f6c4f5ea9 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos.yaml.disabled b/.github/workflows/centos.yaml.disabled index ef6aace7b..420b88392 100644 --- a/.github/workflows/centos.yaml.disabled +++ b/.github/workflows/centos.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos7.yaml.disabled b/.github/workflows/centos7.yaml.disabled index a891a3dbb..ec764d288 100644 --- a/.github/workflows/centos7.yaml.disabled +++ b/.github/workflows/centos7.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos8.yaml.disabled b/.github/workflows/centos8.yaml.disabled index d39fc2b0a..76c476664 100644 --- a/.github/workflows/centos8.yaml.disabled +++ b/.github/workflows/centos8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 6a0edd459..168d4914c 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index 7794a2914..d84cb16b1 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index e8fec2ae1..e857f280f 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml index 3bebff0c9..ad765f1ad 100644 --- a/.github/workflows/debian_11.yaml +++ b/.github/workflows/debian_11.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml index 3dbcdd303..dfbc4ec97 100644 --- a/.github/workflows/debian_12.yaml +++ b/.github/workflows/debian_12.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 1cb4e5431..f3062b6f8 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index e987d7e46..1b0e00295 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_8.yaml.disabled b/.github/workflows/debian_8.yaml.disabled index 37640f59e..b41ead7d3 100644 --- a/.github/workflows/debian_8.yaml.disabled +++ b/.github/workflows/debian_8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_9.yaml.disabled b/.github/workflows/debian_9.yaml.disabled index 42e28702b..451348091 100644 --- a/.github/workflows/debian_9.yaml.disabled +++ b/.github/workflows/debian_9.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 3dfae11a4..5a4f9a9b4 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 3c1aecadf..6b228ca33 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -34,7 +34,7 @@ permissions: contents: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index bedec7659..c74fa2a0b 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -37,7 +37,7 @@ permissions: pull-requests: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml index 351180e94..1b0fa15b4 100644 --- a/.github/workflows/grype.yaml +++ b/.github/workflows/grype.yaml @@ -46,7 +46,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index f7f7383fe..c389f5df0 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -46,7 +46,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 4959ef720..438dd184e 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml index 64ec035b6..cc392b105 100644 --- a/.github/workflows/mac_11.yaml +++ b/.github/workflows/mac_11.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml index 5fa32d10c..0bcd6e6fd 100644 --- a/.github/workflows/mac_12.yaml +++ b/.github/workflows/mac_12.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pypy2.yaml.disabled b/.github/workflows/pypy2.yaml.disabled index 7dbc25c7f..01080bcc8 100644 --- a/.github/workflows/pypy2.yaml.disabled +++ b/.github/workflows/pypy2.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pypy3.yaml.disabled b/.github/workflows/pypy3.yaml.disabled index ac669813c..101b36751 100644 --- a/.github/workflows/pypy3.yaml.disabled +++ b/.github/workflows/pypy3.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python2.7.yaml.disabled b/.github/workflows/python2.7.yaml.disabled index ff3e51dd1..1813931e7 100644 --- a/.github/workflows/python2.7.yaml.disabled +++ b/.github/workflows/python2.7.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index 7d953dfd7..8ec3008b9 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml index 91658c8e9..0f27b0076 100644 --- a/.github/workflows/python3.11.yaml +++ b/.github/workflows/python3.11.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.6.yaml.disabled b/.github/workflows/python3.6.yaml.disabled index 84dce85e0..91bd3e673 100644 --- a/.github/workflows/python3.6.yaml.disabled +++ b/.github/workflows/python3.6.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 04c7146c7..73a0045c7 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index 37ad93654..abcbc4c68 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 9cf18e290..1ce7eb238 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 3ac4c40b9..c2f614222 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 94fd43a22..04a3cf197 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -50,7 +50,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index ea4a967eb..246087bd4 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index c31ec991e..95b2571ca 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -48,7 +48,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 8edea5c2d..fe863f875 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_14.04.yaml.disabled b/.github/workflows/ubuntu_14.04.yaml.disabled index 395184d7b..1442d675d 100644 --- a/.github/workflows/ubuntu_14.04.yaml.disabled +++ b/.github/workflows/ubuntu_14.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_16.04.yaml.disabled b/.github/workflows/ubuntu_16.04.yaml.disabled index 0a55dd214..a9655835d 100644 --- a/.github/workflows/ubuntu_16.04.yaml.disabled +++ b/.github/workflows/ubuntu_16.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml.disabled index d909f9529..4b510ba1e 100644 --- a/.github/workflows/ubuntu_18.04.yaml.disabled +++ b/.github/workflows/ubuntu_18.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index ae5ec97ef..47d0512ac 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml index d5dfb94a8..ba8a549b1 100644 --- a/.github/workflows/ubuntu_22.04.yaml +++ b/.github/workflows/ubuntu_22.04.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 2efa3ac6d..0037e4ba3 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 468d875e3..ab6ab9638 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index 5c3118c33..f2a4ca129 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -50,7 +50,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: From 5dbc2ec8cd49f8990767878568ae893ba7b4dd5b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 14 Feb 2025 16:31:03 +0700 Subject: [PATCH 425/459] updated .cirrus.yml --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 6916e2577..471dc55ba 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -23,8 +23,8 @@ container: image: ubuntu:18.04 task: - # doesn't work properly - #skip: "!changesInclude('**/*.md')" + env: + TMPDIR: /var/tmp script: - setup/ci_bootstrap.sh - make init From 10d0ad24c89cededf549e38c49f209234d90825b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:05:48 +0700 Subject: [PATCH 426/459] updated semaphore.yml --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 761ef0697..e2b78c511 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -23,8 +23,8 @@ version: v1.0 name: DevOps-Python-tools agent: machine: - type: e1-standard-2 - os_image: ubuntu1804 + type: e2-standard-2 + os_image: ubuntu2204 execution_time_limit: hours: 3 blocks: From 0adb1596606a13a7e3ee873bacd76953d4c6d39f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:28:56 +0700 Subject: [PATCH 427/459] updated config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab6c6499d..b22b98dbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,8 +41,8 @@ jobs: resource_class: small steps: - checkout - - setup_remote_docker: - version: 20.10.11 + #- setup_remote_docker: + # version: 20.10.11 - run: setup/ci_bootstrap.sh - run: make init - run: make From c10d63286ad086d0fb4e3b403f823a38aa7699a0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 21:56:31 +0700 Subject: [PATCH 428/459] updated config.yml --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b22b98dbb..312bf1020 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,6 +40,11 @@ jobs: - image: cimg/base:2024.12 resource_class: small steps: + # CLI is too old - config validate breaks in test - install new version to fix + # doesn't work - existing version is too old to update + #- run: circleci update + - run: | + curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash - checkout #- setup_remote_docker: # version: 20.10.11 From d8d48a9fc94d856d27c805b56c2287e727debedf Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:04:54 +0700 Subject: [PATCH 429/459] updated semaphore.yml --- .semaphore/semaphore.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e2b78c511..308278eee 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,9 +22,10 @@ version: v1.0 name: DevOps-Python-tools agent: + # https://docs.semaphoreci.com/reference/machine-types machine: - type: e2-standard-2 - os_image: ubuntu2204 + type: e1-standard-2 + os_image: ubuntu2004 execution_time_limit: hours: 3 blocks: From 3204b856ac322c4c2e3c5b30bbf562484dae5fa7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:14:01 +0700 Subject: [PATCH 430/459] updated semaphore.yml --- .semaphore/semaphore.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 308278eee..e8de9f5e5 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,6 +44,9 @@ blocks: prologue: commands: - cache restore + - export DEBIAN_FRONTEND=noninteractive + - sudo apt-get update + - sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From 2895d5f6ecc4012b3eded89bdb4990f5e41c3e92 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:21:10 +0700 Subject: [PATCH 431/459] updated semaphore.yml --- .semaphore/semaphore.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e8de9f5e5..fdb60732e 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -44,9 +44,14 @@ blocks: prologue: commands: - cache restore - - export DEBIAN_FRONTEND=noninteractive - - sudo apt-get update - - sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server + - sudo rm -f /etc/ssh/sshd_config + #- export DEBIAN_FRONTEND=noninteractive + #- sudo apt-get update + #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections + #- sudo dpkg --configure -a --force-confdef --force-confold + #- sudo apt-get install -y openssh-server -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From 99cf65b7fef1586302c8f0979c4cda90aac1ce43 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 22 Feb 2025 22:32:25 +0700 Subject: [PATCH 432/459] updated semaphore.yml --- .semaphore/semaphore.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fdb60732e..5d5a7ab20 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -45,13 +45,20 @@ blocks: commands: - cache restore # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server - - sudo rm -f /etc/ssh/sshd_config - #- export DEBIAN_FRONTEND=noninteractive - #- sudo apt-get update + # + # causes error: + # + # Not replacing deleted config file /etc/ssh/sshd_config + # + #- sudo rm -f /etc/ssh/sshd_config + - export DEBIAN_FRONTEND=noninteractive + - sudo -E apt-get update + - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew" + - sudo dpkg --configure -a --force-confmiss --force-confnew #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections #- sudo dpkg --configure -a --force-confdef --force-confold - #- sudo apt-get install -y openssh-server -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + - sudo apt-get install -y openssh-server # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build From f21e0aba680245a800caf478d4b248c8e3870e62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sun, 23 Feb 2025 02:07:53 +0700 Subject: [PATCH 433/459] updated semaphore.yml --- .semaphore/semaphore.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5d5a7ab20..59aa80836 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,7 +22,7 @@ version: v1.0 name: DevOps-Python-tools agent: - # https://docs.semaphoreci.com/reference/machine-types + # https://docs.semaphoreci.com/reference/machine-types#linux machine: type: e1-standard-2 os_image: ubuntu2004 @@ -85,9 +85,10 @@ blocks: #- name: DEBUG # value: "1" agent: + # https://docs.semaphoreci.com/reference/machine-types#macos machine: type: a1-standard-4 - os_image: macos-xcode12 + os_image: macos-xcode15 prologue: commands: - cache restore From 1ad7d99584197c2cc80e91d03e6dc335fa2606f0 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 25 Feb 2025 21:38:06 +0700 Subject: [PATCH 434/459] updated .envrc --- .envrc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index bafe8df0b..e4488d51f 100644 --- a/.envrc +++ b/.envrc @@ -71,7 +71,16 @@ fi if [ -f .pre-commit-config.yaml ] && type -P pre-commit &>/dev/null && git rev-parse --is-inside-work-tree &>/dev/null; then - if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" + if [ -L "$hook" ]; then + echo "Detected symlink hook: " + echo + ls -l "$hook" + echo + echo "Removing" + rm -f "$hook" + fi + if ! [ -f "$hook" ]; then echo echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." echo From 6b1938824a0bf6ef7a278c2f08061801dbae7ad9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:41:33 +0800 Subject: [PATCH 435/459] updated bootstrap.sh --- setup/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 94b10159c..d7bedf92a 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -45,11 +45,11 @@ elif [ "$(uname -s)" = Linux ]; then if [ -n "${CI:-}" ]; then export DEBIAN_FRONTEND=noninteractive fi - opts="" + opts="-o DPkg::Lock::Timeout=1200" if [ -z "${PS1:-}" ]; then - opts="-qq" + opts="$opts -qq" fi - $sudo apt-get update $opts + $sudo apt-get update $opts $sudo apt-get install $opts -y git make curl wget --no-install-recommends elif type yum >/dev/null 2>&1; then if grep -qi 'NAME=.*CentOS' /etc/*release; then From 2efb7e7dad0b6979452cc4d229e50e72e32bdf9c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:41:34 +0800 Subject: [PATCH 436/459] updated ci_bootstrap.sh --- setup/ci_bootstrap.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index 0b52378ff..b01a77d04 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -72,8 +72,9 @@ elif [ "$(uname -s)" = Linux ]; then retry $sudo apk update retry $sudo apk add --no-progress bash git make elif type apt-get >/dev/null 2>&1; then - retry $sudo apt-get update -q - retry $sudo apt-get install -qy git make + opts="-q -o DPkg::Lock::Timeout=1200" + retry $sudo apt-get update $opts + retry $sudo apt-get install $opts -y git make elif type yum >/dev/null 2>&1; then #retry $sudo yum makecache retry $sudo yum install -qy git make From b13623f40359ddd19ff96f6c2ae78a56375b3222 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:52:41 +0800 Subject: [PATCH 437/459] updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a435bc5a5..2a53008d4 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,7 @@ apk-packages-extra: .PHONY: apt-packages-extra apt-packages-extra: - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y default-jdk; fi + if [ -z "$(NOJAVA)" ]; then which java || bash-tools/packages/apt_install_packages.sh default-jdk; fi # for validate_multimedia.py # available in Alpine 2.6, 2.7 and 3.x From 0bba44755a7b874dfc582aa7e15d59c7a94d0986 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:54:28 +0800 Subject: [PATCH 438/459] updated Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2a53008d4..d4434ca84 100644 --- a/Makefile +++ b/Makefile @@ -189,8 +189,8 @@ apk-packages-multimedia: # Debian 9 Stretch onwards, not available in Debian 8 Jessie .PHONY: apt-packages-multimedia apt-packages-multimedia: - $(SUDO) apt-get update - $(SUDO) apt-get install -y --no-install-recommends ffmpeg + $(SUDO) apt-get update -o DPkg::Lock::Timeout=1200 + $(SUDO) apt-get install -o DPkg::Lock::Timeout=1200 -y --no-install-recommends ffmpeg # for validate_multimedia.py .PHONY: yum-packages-multimedia From 72d01f4ef7dc4646eb3a35d665dce0936a445b65 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:55:27 +0800 Subject: [PATCH 439/459] updated Makefile --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d4434ca84..1d98ac524 100644 --- a/Makefile +++ b/Makefile @@ -200,9 +200,10 @@ yum-packages-multimedia: .PHONY: jython jython: - if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi - if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi - if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + @#if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi + @#if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi + @#if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + bash-tools/packages/install_packages.sh wget expect sh jython_install.sh .PHONY: test-lib From feb48b7a21ea137bc18ca92344cdb8a5e75d98bd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 15 Mar 2025 03:56:01 +0800 Subject: [PATCH 440/459] updated test_validate_multimedia.sh --- tests/test_validate_multimedia.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index c86449aac..3493bae6d 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -32,7 +32,7 @@ if ! type -P ffmpeg &>/dev/null; then echo "WARNING: ffmpeg not installed, skipping validate_multimedia.py tests" exit 0 if type -P apt-get &>/dev/null; then - sudo apt-get install -y ffmpeg + sudo apt-get install -o DPkg::Lock::Timeout=1200 -y ffmpeg elif type -P yum &>/dev/null; then echo "WARNING: cannot auto-install ffmpeg on RHEL/CentOS, the 3rd party repos and deps are seriously broken" fi From 55835218d3367e90d76d6a0614727ede7351e1f1 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:38:30 +0800 Subject: [PATCH 441/459] added plot_uk_marriage_rates.py --- plot_uk_marriage_rates.py | 158 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 plot_uk_marriage_rates.py diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py new file mode 100755 index 000000000..6b1297417 --- /dev/null +++ b/plot_uk_marriage_rates.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2025-04-26 00:37:28 +0800 (Sat, 26 Apr 2025) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Plots the Marriage rates in England & Wales using the latest download from the UK Government website: + + https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/marriagecohabitationandcivilpartnerships/bulletins/marriagesinenglandandwalesprovisional/2021and2022 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import platform +import subprocess +import sys +#import time +import traceback +import pandas as pd +import matplotlib.pyplot as plt +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +# pylint: disable=too-few-public-methods +class PlotUKMarriageRates(CLI): + + def __init__(self): + # Python 2.x + super(PlotUKMarriageRates, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = 0 + + # def add_options(self): + # super(PlotUKMarriageRates, self).add_options() + # + # def process_options(self): + # super(PlotUKMarriageRates, self).process_options() + + def run(self): + if not self.args: + self.usage("Provide path to datadownload.xlsx") + + file_path = self.args[0] + + if not os.path.isfile(file_path): + self.usage(f"Invalid argument provided, not a file: {file_path}") + + log.info(f"Loading file: {file_path}") + xls = pd.ExcelFile(file_path) + + log.info("Parsing xls") + # Read the relevant sheet and skip the metadata + df = xls.parse('Figure 2', skiprows=6) + + # Set proper column headers + df.columns = df.iloc[1] + df = df[2:] # Drop the header rows + + # Rename columns for clarity + df.columns = ['Year', 'Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women'] + + # Convert data types + df['Year'] = df['Year'].astype(int) + for col in ['Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women']: + df[col] = pd.to_numeric(df[col], errors='coerce') + + log.info("Plotting") + plt.figure(figsize=(10, 6)) + plt.plot(df['Year'], df['Opposite-sex Men'], label='Opposite-sex Men') + plt.plot(df['Year'], df['Opposite-sex Women'], label='Opposite-sex Women') + plt.plot(df['Year'], df['Same-sex Men'], label='Same-sex Men') + plt.plot(df['Year'], df['Same-sex Women'], label='Same-sex Women') + + plt.title('Marriage Rates Over Time (England & Wales)') + plt.xlabel('Year') + plt.ylabel('Marriage Rate per 1,000 People') + plt.legend() + plt.grid(True) + plt.tight_layout() + #plt.show() + # + # doesn't show anything because the script finishes before the GUI event loop + # has time to process and display the window + #plt.show(block=False) + # + # pylint: disable=line-too-long + # + # doesn't work even with this hack: + # + # WARNING: NSWindow geometry should only be modified on the main thread! This will raise an exception in the future + # + #import threading + #threading.Thread(target=plt.show).start() + #sleep_secs = 20 + #log.info(f"Sleeping for {sleep_secs} secs to allow you to see the graph pop-up") + #time.sleep(sleep_secs) + + image_path = os.path.splitext(file_path)[0] + '.png' + log.info("Generating output image: {image_path}") + + #if os.path.exists(image_path): + # log.warning(f"Image page already exists, skipping recreating for safety: {image_path}") + # #plt.close() + # return + + # doesn't solve blank png + #plt.gcf().canvas.draw() + + # results in blank png + #plt.savefig(image_path) + + fig = plt.gcf() # draw before saving + fig.canvas.draw() # force rendering + plt.savefig(image_path) + + plt.show() + + if platform.system() == "Darwin": + # fire and forget + # pylint: disable=subprocess-run-check + subprocess.run(['open', image_path]) + + #plt.close() + + +if __name__ == '__main__': + PlotUKMarriageRates().main() From 1a0723e81e31574595e06f5cb7f85038aaf033a4 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:39:20 +0800 Subject: [PATCH 442/459] updated requirements.txt --- requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.txt b/requirements.txt index 80ff670d0..05d204c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,11 +36,17 @@ ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 +matplotlib==3.7.5 + # Python 3.5+ #numpy==1.17.2 # XXX: install broken on new M1 Mac Python 3.9 #numpy==1.16.5 +# for plot_uk_marriage_rates.py +openpyxl==3.1.5 +pandas==2.0.3 + # requires pg_config to build from source #psycopg2==2.8.4 # XXX: install broken on new M1 Mac Python 3.9 From 53def06b018590b505dc43379c0b27e99d5e1fd2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:40:48 +0800 Subject: [PATCH 443/459] added uk_marriage_rates_2022.xslx --- tests/data/uk_marriage_rates_2022.xslx | Bin 0 -> 15337 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/data/uk_marriage_rates_2022.xslx diff --git a/tests/data/uk_marriage_rates_2022.xslx b/tests/data/uk_marriage_rates_2022.xslx new file mode 100644 index 0000000000000000000000000000000000000000..54efc33218a9d57c8f26797118393a120865a840 GIT binary patch literal 15337 zcmeHugGM3^)Wj2nYxaun~hS!31!ngMxq{K!bpw0so^TVrT1YX6tOA z>S1r@q(|>=V_mBzBhw>@+z$5?TBTu8PaP(yg2W$9UA*oxDxY3>Z~V68xWnzsXOgMV zy36?30q>Yw^Y|{n*M&_4)U0*nV znNfCN*;>ks*Vqzsm~ER^Ruk@fCs!U8Jj_w)7yG08ovGzSZWeb4J<}US99$5``p6+h z6Hf>d_7GuufUKbd&s1<}JWUOwwQMhGeS1Kz&zHs7ypyagNpX?09Ml!6fmOiQ;p^Bp zmBg~cmqbhTZyJi7hi?>CMsAuak*Gws_fV}X*J~7PTo{;<$(X;-ey)qmxF1W-#ivE^ z)H(6|fWAET1OF?-5T??uuHw%0Sr&_(1QmE?M1~7;C5!XG3b|<&LXe~rT;u7h@tpJ& z+$2E(R6ojoFU;@wUYXIh^7J+wG#vV&)2eeIT>&!Y*@ydO4vPhtX2K%xDZy?1IgsjMtc9-IcAxpRBO30d1(Am)VTx11{O zz026jjPyy`K|upZG1FLC;no(Jo4gG>`>LXLQw?v|8lis4+7PU)!6@CX_Tk`wVOs z9y7~nofbe;`M5u1IAsm!`Yy27oqaVO5aVG#3%0>2qZ1ukn*H|Rl9k`d$}P4OKbL1_ zauMvZ^RYDBuX!2f_6KEe0w{`NzqBT%q z68cV;%yQ9fBsVomoKq{E;n% zUSP$%fki%#TDrI#Pg0U@wWIYNBRcMtk?_@F!7f`Fx36Jnk&8>>MLDZmbRT)s$l50U%zGZeT0vMybI9Xkd6rBcWqEr z6K6QAzLls=QKsV3s~!VEV_d8bHsSdO(t`7Soi@I`B6ahar0@ey;kVj45IFi(i;r12 z0DnzPPH68NR5jr;&30ii}w`@+B52?HQDIrqEe)HIe38&FDa@R$8e601?t-;JO zazQqnTn7Az96g(S3~G>5xLov{)*RPxTcxmACIgps#|8PRQKR!;$EV05z_)jLMS91; z(BsJj^Jytw5$6(eZ*&fNN?353_%GUXEOudX8C=F=+gl^lh@{GihbOjQ-asVT7DPtlxjK#|;=vVK378N9#pZe^tW4qL?~~lvEOJMK z@<{j=upYek{dSU;hI&?aYV#GpHcP#1C$!=aG4c9+mh~o7I3L&tvH>m<&RV5-cZenn zen+1Sd1ekj($Z-I-}3_4vJD&Y*Q#yPPz3#P?K}4iTicEt)u%Ji_hL1w6kq)A*T6c2 zuGn~3I6HV)?9W}ev$L?h5=IG6E}R0lEaLlj2L@`i9`1SXkE?<&V<787LJkj!R1S#} zN<02_O|KQarNjZ!pD6?g2+kjC`VR*D>&pJchOLQmRy~aHA*X?l&|CL;WGl_g2S1ph z+HA3pEb#z={jOEBHn-O-*fJ`V3*@v9Fj`2#V49?-8w@Qq=EgX+tz1ET1h_6f^tI08LU1w9ehFDjN(!LvXeJ5O4D=!0^ zTbh$_KtvP78a=;Wgu}}-ZF({e8 z^?@TAj5KuQDn@3pG@qhWAYe%nS&Y#E6rI72UesTZ3wyB{*4p-P&~ZB9KLsy zBtDLS1IxVz9?;?u4LG9gRm#j2%=w799Y zEa~&~L(pKUEOC{Uw0l8+l2Vs0j8X^dU%&0}>t|00&9|J;$D(o3F~kwa3Qs;%u1^OO z5GW5I7N4uGH;u9t^EU|ULz<2ud;@J$UyspEOjKLnS?9Epf;?iCn}Xm%p`G%%R_SY3 z=>qZ+c9%4RGD3JhCFzGDjBFSaCnOP#q;2EeTlw(0Od@gCuR@RY$(el9=pE3XQWj2{ z-&@Us8q?wK>+_*zHh4{obo*xZvc zlSCRwHaxBc=Tb2lGu$Jz=51ali9%zldk2%B2AN@McIcY7CYbsA?bh^Jb16s&$?lF2 z=^@4&VihV?4P#!gKCxy@bKV7hN>J`byw)QoEdd+f!%}8Sm45)eBkt>J#~~e=;<*a60WXaq_t zRqJiF@AqI5bJT%v2{G(8`|G-skZ)856sjKWZ3!h;geYN{YvGukfsnS>$d_h+dUfiH z(*4DQxVMy1YLhb8j&j;hI_#|(Y7v#s5*qb+7V@|>&W=Y`fz;EQW%%h8!!Hy0^%AMQ zWQ$J&|CmHg$cSxddruiY!*_%fULx(oy;~fwfosX=MD_*YEcBZab^p`{DWN6#C&X`*!~ruw?WAJ2yW63F+p!vf&JHJHIq?!O?p{ zw?lmee`cvz2?c3|aG*7T0p~f9^Yd@+>Kr=uLm>(_N==5q`fqvAnX@^DbpZ?#K1zxg6k#r6mT>(wCQtpg~kFo#R3A>XUa+c=n&SA&j* z6zi!FLoC`^F>ngqb5VDf*wdqD`MVxjc+om;%d`TB)^t2wEB$9M?#&6{I>Y&3nEPAw z^qxkW3ruSXB1Q*u#LI(7qjj0lS)mu^+nG?j7v{Sc#hrb6j-GU&POCx;!_ymz?v>y? z@A>*@zB5-vU@@HxJ`%icES8sNP%s?h#qY=*$~`${r!~3nVZg_yOMdN;rL-gVhLN2@ zm_@Vly)AX?xc6`aLadV?abV_X3qHcFbp`Wg*)TP8=Aqgi`8@sy*JYLgIC`Q{>I~Tg z3^$JUauiY;*^9(^*-P$|9cJ!fgj6mb#r!#?j}5h(r~-WMc=RZgvsYNIaCDt)n&PXdp~g5 zFM#HO#rA?@K=G;B_bxA+bY4MF(zoiaT16NAAgpmzV0^OfxjFhSFQ0T?Nl@~s>aIy8 z;}RT2DcvV4H=?Z~yM1qW@DeTd>gtPaY+m$Q6i|kLe4$`LTQbU{%@db6(sRxseTvgAkK>9QqM94Ws@pMo zHnhG6GRN(biR2Z8o?BPs!>yZzWATGV93MK;CLEGOx?jghh{)zPI8I8Oz%#!bVzA$P z-^^M0fP1YU>0wI`NXnH8dbdtzWyvn{LU2Qbv)U`OH)wiUVg`h=a-)Q#S6FER%_kvc zEuGni^hCRl2=XTalK`<+)0~dB=f`1OX8`0yoIqzbjh$oh?H05`vp2gZMgAjoy9r8z z#&l(`kcKhimSrOD3BXXA8|CE12hGvivs1 z_OM1DoX252XEP|43US_)hGAU6mZN2o{JsQX*=YlHV%~J$Ld0g}Kx~bkByO&m8m(tn z_HtVhz4cQRtWo}S-<2%q6`*Lyp{rwYX;b(q+Xv`LCsNzFl{w5J(M1bth`QlUJ5^uYIRp6MSvN zaZoM~7_V_}k^w!m%-xwpBqp)FIDP(-Mei%2f$qD7ukKi^@UUrQOVP+%4z!C7gAEG3 zjjoJqKfupR4?-JSp_1SC-?TF#H{4!Z-4$4I11IIQDqaLI?nf*!q4khQPGVbgs_{cg z!*UW!@Rq2niQr4eemGkqfoU(H+NSd{J<*u%<@+NC4 zCJn`}T1})8lOEqtBXzx9==iDC{D#)N5hJPJDk*trw;x(dHXBQ&?<>b^Il0kC>M{Fj z;pY41lQA(jjf$4{83sk^4@)2c4Ci9F4qn$h&eyjOG`Rx5d7d>&2r)nqUaWHJ2S@-@ z|2r&oTiwP@htQ zrYtD@C~U`66trjz}wI4N1Z6(kcQQOsW1-onv7$Sh_1I-=MN!3=x@zZJF4 zO(vA$TB642g9(Xi>@?@}UwacdAlXftjWK|SAGn%=K~tz)jzt=OqaR?b=;gyqOnQWn zB<<(*xm96q?$apdaLJ)a9Hr3@(Dii;<-^46A!_r*VHv7Dyr|hTk^+U z%mxOGN%c3P04OaH+ni_SCRsyo%}J;8{VvNXBA7sJuR;UxV%>qu!$<9o`l(N#?^pwp zit4{{A12wx8X+j&&E~dfoT`mrtWFr{)imLk#01ahsQOif5W+XFi@{Y2QSs~cgl3w5 zdV4sZgKnVHGcR+juh{UzN>NnGu`z@`xX#k8Y4Npb_g*yGDtw?&TRhdcQY=F0k+O=c zCJ6^XJMx5?ALKolGMUuoP#?U)1k9Jl2uiN?-X-#B7*x9u)HjBWs#FNlYB!bq{!c}u zAL<#I3zGBc0uiTA$hh*y?XmVybh#0c?yq>=AUxaq>`Tf|t?J@Y$R7Wji3r((~ zaxxoZ@cj6QxoXhy^!?zBV}FL?th=}4;dbO>$MYHYW!2{LYle28^Yi@#%r&3OU4NOitfK82KaKpMeTw_A(0h#tWkCxfY*{41($F5FK>8R2_FO;c50m zw(@pZEi@CJHIYT9R*F|cE}KNUV!3gzm!yq+|8AcpyOMCx6y6d(5odKmFI@(5y~~oL zv8lWeRb{q=@BuZvh_h=-rK4IT62G)XBn&Xm$Gj<975?J{xiTU}L zq}z-rXz_e>)uyti*lL!~+mzG@*bheEu+M=0k$3b;N6K8HOlqb zL_I%HUEG<_2FdJjhhE|cb-==JA>I?{^|HsyIPOg~MPRYS@wtABQsi}7l#L(k^?id# zEEeY$D23@Av%n+DCxWN=g=x9rRekrPIY}erwq;r(McAF$7=CqYWNI5BrgK15gxMl* z{kOW&a26i!i_b}mEm$hfk^NX+4g5};6Gki}yG@fCB!)@&N77|9gJWu!8HS){@g-Js zRE{5EXw0t95^bv9+BpQcV^D^ptQFkih^|QMaRM?_Ju{;b<<{B20EF8S(^T`2dyMfd z9a4u;Fj{XvdSaY->)F%z+SLpg8m#giw?kliTVfMkHC&A%D_I5b8il~X!eg!kguQ;! z70oQwwmwNr|CY{<1dcAQ-iDU29o07;{)OVfZJ59f)CmeQM(6B4kvpBzY**C3=G8SJ zt(E7ytmRJo_%f$C4>c$K*_-G467y+&{z|hktJDka{>!UNAFP#CMUlIcA zBwn5kjmCRqe(kL^K^nqz2V`fSL3idZQj3rkETt@YLjMhw3j3#$zBT+ZvIV^EHo~Zw zHgN7;4wM2-O$Hk1=ZkvX~@8(-SOmQVflX6EWy5^;rW_7cL zSA3mRpdrLkSV-&2J*4MNDFMG~Q&{03oE>(8vex^-j5Cjz{SkuLiyFyq%M-}9lSMro;*g%$>0VTS$2#zmZy@2rAC zU@-Hmwu>O`p>*cW-SBejtU0usKd+~~ALl3KJw;)=kEE1$Q@DeP?1mA`30hGwYpq-q zbkDw2*3c``-{=x?j5?521C?`8>Pv%NvD8N%t8@=ssFBtUYss`m49Xkgobt-Hi7A8cHkC_0CXIH+pm+ehqam0&$K{++L|3OE#RA7``ope zi33wb23Ar;rCeB^HY2}S3jU^ghj1_@%z5#-oh3G?c)Gccp^N{X-_LDrg}mtmLc*&A zx5)J1Mk6b*m{xUYo86_I4{)+96h-`nOjjz68+ws*eqjJf$^x)747YF1}?CN2I;wS2%<=Y=sXOv2dvD)hdN_b_>WwRJj_N@zXo zkt|Mo)p%qd0FUa_a_o4;uD+76Gq=@!l=KSEgsedrB_$$gT0GC0b^jK z8k2ngz)mL6t#6Ow^=T~(-B|AW>+r5=xV0F)Y#{8cG`M4y>^sgThAym7L>$#7HS+Bt z@JWZ{^+_%nYH|jqGGdvux3v^_AYfi?in!Upmy4kah2nCgq1ej3Mu{YZGC2)~ zE`FNsg7{obO~FQ{iKH8iL$)os%rO|S*lO+~pIQ_s+@f%#33ZAid5 zMJ^zzo1VDefomNBH$p%Rw~C^QMRM-BC$EBg!@xa8>cb&QI*{;)SZcb6XuB` zvt`EH&>3*^h0MMq+`eyTQzPD zn|G=306^$SNDtABNtH=h%v+rUS$6P8{nt5sfOZ(;(>LdQ9N8Z2xc#OlV_RC=J8^_% zTlOxPfXN`QAN@*uMr(v2CFF#32xy5c9y15U6Em-nfR51}6)q5f?#^xtwf zJOP)92QUHov%>asF}yQ!G&5CkcC@g4`)f@^s>)d{2_m;)-VuiI@g#4HOkqjoChjCd z;DjnlBbj(u(;bC2*Ek;et!Z&t;8IQC1=U{M?zCwoX+T>UD)`sAB~kA*fjLI4y=y3) z={dMNN&t)AV}IA0FO45QtM+&%GK44Oh_6u+Rjp%=%|WNN-E=w_{`N4ik8+@Dq!BI% z*ET$lZGmKboF!tB>YewgaUne%WZQIy=ba-nI%WyI);wUsbO@h)O1FeDnD6tQ_UWBq zm8NiNU-MVxJ!Jh#ZB&T}cqt^0zi&Mp_IChI3 zh55M_OX>4$X71nhcC{q9O}O7P!bDyOrubbKJx@3utO=uW-f0G!9Z=Q*Ih6bh;q zECRxNJEmbXQSfV}cFu#4fzgvEz4G}(-Yz!%X0(W{s?#7N>lP`;ugrB& zOxK5QMs%@j@Fx*>uY|c^co7v2l18;%aO1m0sHKmw*XWO4HF##70D%2CfIa=5_cN|J zVjS&EGg$Dp3Ff=1?_P^^9YWQd2g~TM=y+p|rs><$z~$wo!v|tDU&B=gE0H~7_4xKK zR7b!19LvA6AQkSxICL9)D>UCSU}cS^hMqY=o>r>ymK6zxOEYn$IOCV4&{ zB*hhL$e$<4H7Is`{@z&V(CsSA$_Ejm$mZ+F#nr2S8-YNIyfy=%=6k?K@y7_*1DA@k znWKuCv-8h2!|>uW6TPv8Ev&hu*`SG}k)k1`(I9mEiQtz=;`g~>#AuZI-GVTTUl^-a zjn<8jvqlCx2RnxtBbI_lAqR~OCyY$Nrisw-uw`M=M4Q24>4-q`jI2#ny!CX!u^@;X zfgSkEEd5y_$Azwan**F7L14KM1$c+4or$8OoxKx-iJha_&uuO;Ue0dmXEg!*ju1Zc zaBz4Yiqyg1b?-AM6l-6|3TwDr-Ece+lgs0P;IA%0c5WV`nbafR)wReGC{Y<~+I zT0(%?dg6Tl@!XMxSGJ##mhgUU?P^zAUF+2Fm zKt#ht>r&7T()@&kb~l*%h5Z{;+Wy1|mrf}$XUSRucCg&EY?{^@)Fs1B%xI5^fTgW& z0}W)EwuM@r5ZvsC?5rQs_XZt88|wrJu*EQ_K4k>(%8NCirIj^N#8q_@BGDgguPZ0m z`42=h;35QZ;y=70dfZb4bPHM1pxN$4d=6>1c~<*wXQaW=oxX2tR_yhq_Mm}B7L+^R zC#lCZChoMRrE3R{Tz-jydoKob!PN$K* z{jXX^tenEH+K}W!DCIR9wj^5Gl``{~?B zK#bjbl)azdUE@Z&1&&{-s6H%oQiLI^8VN@H@q~#dv?9XvC6s;wZh7}r^RzUmV@2!E1)(1W~4uDeNw1?@c`(`eW3g!n_}YPa*<#%_rn`H4U0S!jZPDgFUWMTqj3x8bjvlygWyedX8F#T8ezE zV=Zo`g;q=bixJiwm>lh4^gzp&lN-|g)rt`zR!pLU0PK7bOTAoR2q%f}O%*%ZMil+1 zn}%Jr<_0A?EBf^~Q(F&Dvji0HLp{5kNz8e6;Ya>_KE3oM7z!_B6}zPN$40&Mt1j2| zN*lY3!^6Iv^Yi}dPh-|C%eEB_W7f8zFEUj`EN@Uk@yu?`AWX5d=0+cLWgTyf3|XJh4h6aY=zcTiEss%eg0tQbTUn)< z&O^MJWaa>P&A)n|BF1~(izVSjbIwbsVr9I9IhO@18Y`$igp!)ahm%KXZpeqm50JhU zhuLWf&slEuyqI1Br=YP(B1ZPg5$+EffawVt=V;cH;ffGLInt{NDv8Nb0Pik5eDkgT zjJ(Yr&#kEna~h7huI<+OqzUb9ePbhQRIGw?e~@>H-{csCms3wxR#v~lv_mgJ!71Kh z%#s54F>@%s`?K5D#}kbuXxB=)1AXJA!m+SsCp%WBW5By`WNbNIW(gJM3q0s^X0()w z(a)hab{p%H@!aoNNeKvYZy};6wdi*vQaRFdwFv0_2qk2(j8Ka854E%!)6d(I#{m|C z00%XwvQg#18c3o~Tki?T;J2y7{m9;9-|m(~A8oJXA@flJZp~x%goR>1kIpb2l*^?u zyYci=-mYM_7;p)K6l4&kNvay;eq$}H5Qfv5)D4}f^*)Vspc+97m3vCFc+DzTd3*|F$o=zpYa;Yx zFuG-yaH(%7J=>m0@0HF$E*Z%CySSWAXnPV!+Xp!!kt;}&%nHa+6Qft0D#*$R_)I6E zTE2T?OsG8J@oistWA#j#Jj4N zvWgMSjF1{`rBa79p%{epiBTcBw63WOs|=ZNqu2s+!4{AJt3FRUi+Ze@TgLWi?FkA$ zTODB@H?mQVy}?g3|HC=1M~rK@#@$0516K`Upr+|TUWVW49WI!5#eSq?7;kc>GnI>g zM8FOe)a`Y62$Jzz-aI1z8-*Pwz2GDRX`BzRN?N<_DEn$io@{LW&zI{R`bgS1wGW(J zjAL0Pu;tvY{KV)Xo2@CThhnA6R(Q<8VK`&lnaU1H0o1PETrsIQ1O@f^wrljap;>Das{^w*}>IdYuTqXn5 ztHC>;Sv!*;XE<)tQQ$=>#B0jrHRE)gx8=;=M0thUct$>;8x>HEss@nRu7gXB6fdf9 zTWhat+CIL=@22FeJ4!WEpbZ$Xe+TDS@bUFRh=GYmVX38v%lFtN$^;Zo=f-W2<&vGTAr997ILl`33T+c_N|SB<_@ua$ z3g-I-E8f|tpw2An;>f=;%;d}J8_t&)6Yj~dp)8yyFKsxP z`kUa;!qpBNYAvK9)p__zU%l?UlPst)ZxFDq5Am#`#y;HWMm_?zV_-W0w$qE?Q}ROQ zMoFptr|YawmZ}WsmBwneor=Zru&QcY&l#WV4rheHRrNPwU$Y}d*f}gMO~O&}IML5= zUv<2yd;;ck{^=UnLL80t6&X(a}laKq#6)qaXc^29pU+1B<@XlMUOGc$I zZ=pq%RhnNx5is#v6$W_aG#Cvj6hJ^{wMBg%bL;D2q~aBNv-9h> znLe=z(Fw}CTMhJ&9In&n`~owCcDGZlO0#6Vjcs;%B6z0KyhtaO4%d6pCI-#Qg^G2e zi{9vEdf3ax>f9g*aAvC6D#dxIqtHV{OhtnB^Yz!UL-xWVH)_k6PSZcY%MTCc13z-x zEfDCBfBLB4^-&XnYr;*f`*{vcg6iGK4qqpWx=D#bw>lut_K4!uysY(G(+rCi!=sJU zhAkeTI{#Fdz{aY+7I@`k>Y{1FBud^q^^uwgs}J|9#vERK-X6dLreknfvfp2*^D~9lA$%wZ}eJ zR2PxEkxGe1v>vWd?MbEF<#kDNJ|$*b(ejIRwr2S@CM_mQ3v&11N;9gq@%|EH6dcH&Gr0C}j=^@ZSHwe}cxlK7cX7L#?;0^CT{3j=3Yp?YnDX zw1zjX*Eut@^jWB+QQlo)DA3RzF}=EdDp(QdE;~tCP+^@=WuuIY_9T=57uY?W#XygU zYRja`^QT}wY60Z&ZCDr5o^7L*=83X1h8JYc9le21rjf=PG0nj%OJt7W)^e*OJ#f*p zbAGZByf!Kku_Ge9gPOTbl|FxFM+kJg5j(+P9416rT+yV|?e_kf*f_`@LkZPt=-!Z4 zvJBU3bvfX-^0x*4rxoZXQ;_uhFIMo&;Q!AG{$lt*E3gCx$588Dxu=HC1{qh3(&5uG z*xT&laBVmeQnDr^n$i|)zD1ufpm?N(*BIDBy;5coSiC+!v8y4#=I5^tOm^P>xU2^6 zGd8h%`{CWXKYXZbo29RPy>hbqdAOPq32ut$l)jZK8q~;Z@<=mgMMQP!REf;8mPM4y z`U(LTU$z_{7PWcYLA?%87tezDhYq=(b$o&$I?((Fl-!kW;SJQCQ`ztc>x(esPPmY_ zRys*uLJd*A_mzJ?KJaJ2+ZJR@8Vgi#3E2M0^DIAk-a48G5y*Iul>t);bbAwJ1g>PQE_kyN_|oT@U1)D{cncRwK8{B0P==Z-%Xxt~ zBv8=ML4MS7pP6Iwmu0{2C8p+e{a$BL*QKd9g>@9DYQV(T!ZSO=rCT#f935H0bgW0Jdtrz zqI4i{M=xdlo9oFZmcn|KkjruVvKKQvZ)-$ z_T|G5#3DVkT$u};WbTQmbGwnW}p1 z&6OFx-`pP8&8k6OM+(0l&Z==33EMA#=;t+b^dwVXq&2-5tJc$X>=>6w=ru)4?L+Jji0Ak?M1c5kEY9mqj)3l2HW8c zs8x{n0m*=hu&ocmk_`b7dFaj~SacpJ7@>*FZp!EB)Cxn7skfMavz0$N&+7s#|9|7W zU&iu(oCh4gKYxC6p3yJPbNN*uezzU~^cgLt{njlD>CaG^OH0VutHQS1=8iBTlW~Vr z4or(s%V)m0XO`dB6I|BY7`QaC88))aa$-|%&G>WcIm-E3PxC^D+HH@aBs`$yuA~NU zYUr-PnN@tDBj^)Ej+~#9Q8|U=`*C;Du>nkh{y`a_V6eYRQZPWR{`t_?pQ`@x{x6S! zDa!mE;N_D>zX84i*XCcIJ9;Vj^0}Meg4e*P@5as1Lw3pp@Sv&s?AOH-ofB?TM z=`TfJRvLec%HsYZ`m*l$viC1*Prnho@n4R?zgD7N0=_Ke{08(T_yzc)kn^vFotJvP ztiAjOB_aHmO3X{~m%0Am;%7vE5&uu-|E2WHWcY7sM3R>g;jhs5CBVy&`8NO+^-F-? z0rX3hm!ary6ik|zD8D?`mjEwAyWapXv@ZdEb>qKL{%Y=j51W2tfPkF;4dvg0sGmLh zE7sqg@4sT*(EqhZzp;M1;xDy(>B0R*duIGA*3Yo+*XH&T;H3xk8=#ZvCBU!Y`B#*e zzR7QtGUmUS+8@E@Pfz9LnEZWn{w)pya>Vl2Zu}mT-&?h!3?z^MK|nBozw5wbvKMSW GKm8Y$>}z%a literal 0 HcmV?d00001 From add60aa8705f2481287cf0fe9b77c0aa7b4e78fd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Sat, 26 Apr 2025 01:42:05 +0800 Subject: [PATCH 444/459] updated plot_uk_marriage_rates.py --- plot_uk_marriage_rates.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py index 6b1297417..a38332443 100755 --- a/plot_uk_marriage_rates.py +++ b/plot_uk_marriage_rates.py @@ -68,10 +68,10 @@ def __init__(self): # super(PlotUKMarriageRates, self).process_options() def run(self): - if not self.args: - self.usage("Provide path to datadownload.xlsx") - - file_path = self.args[0] + file_path = f"{srcdir}/tests/data/uk_marriage_rates_2022.xslx" + if self.args: + file_path = self.args[0] + #self.usage("Provide path to datadownload.xlsx") if not os.path.isfile(file_path): self.usage(f"Invalid argument provided, not a file: {file_path}") From 76d07c239817b67971c1e96aa9246abaf6e7b133 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 6 Nov 2025 23:18:46 +0200 Subject: [PATCH 445/459] updated README.md --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 22741ad41..40045f8f9 100644 --- a/README.md +++ b/README.md @@ -637,24 +637,35 @@ Does nothing: [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) -### DBA - SQL +### Databases - DBA - SQL [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) ### DevOps Reloaded -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer-templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer-templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) + +### Monitoring + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) ### Templates [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) -### Misc +### Desktop + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) + +### Spotify [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) [![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) @@ -664,7 +675,4 @@ The rest of my original source repos are Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). - -![](https://hit.yhype.me/github/profile?user_id=2211051) - From 58f3505733d4fedd2abd8baa6bc69a64cde89580 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 04:09:59 -0500 Subject: [PATCH 446/459] updated README.md --- README.md | 55 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 40045f8f9..e8cfc70d7 100644 --- a/README.md +++ b/README.md @@ -601,8 +601,8 @@ Hortonworks guy Jonas Straub: ### Knowledge -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) +[![Knowledge-Base](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Diagrams-as-Code](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) ### Containerization -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) +[![Kubernetes-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Dockerfiles](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) ### CI/CD -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) +[![GitHub-Actions](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Jenkins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) ### Databases - DBA - SQL -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) +[![SQL-scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) ### DevOps Reloaded -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) +[![HAProxy-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Terraform](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Packer](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Ansible](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Environments](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) ### Monitoring -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) +[![Nagios-Plugins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Nagios-Plugin-Kafka](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Prometheus](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) ### Templates -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) +[![Templates](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Template-repo](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) ### Desktop -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) ### Spotify -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) -[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) +[![Spotify-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Spotify-playlists](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) The rest of my original source repos are [here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). -Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/) +and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo. From ba4689ad10b7936ce3b07791671bb0c2890e2b14 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 447/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index eb10979e3..d65ec80e0 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit eb10979e3572ef37b35bda3653b25d7a45e25169 +Subproject commit d65ec80e001bbf9d15f267079c101ae01578a183 From 572228d476a5cbbd8dd6dcf34217b4d5c3e82738 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 448/459] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index 575080b54..fa8985e60 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit 575080b548fec8b2420520e161973a861cbbce30 +Subproject commit fa8985e60aaf5bb26f812cdeeb20b676684e3964 From 8c8d338d2cc4b7ef9282ebe9655a38eb31a4963e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:05:23 -0500 Subject: [PATCH 449/459] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 2d12574ce..344a446fc 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 2d12574ceafbcb13f559dfa46693024616b8e619 +Subproject commit 344a446fc8588cefe7b812c1fc449a484a27c57b From 351e28284fb9a24d1d187c213c5f090ed99dfce7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:12:41 -0500 Subject: [PATCH 450/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index a97248860..22c6dfefe 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit a97248860172c6bb3534d08ad38c4cfdfd09c306 +Subproject commit 22c6dfefe61703689e7b09185debd3010d93d9db From 824267aa84016f4cca4b4d6e3b971ef6a91d5a62 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 05:12:41 -0500 Subject: [PATCH 451/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index d65ec80e0..9efc29299 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit d65ec80e001bbf9d15f267079c101ae01578a183 +Subproject commit 9efc29299c878e8b781d7714eebcd5c06cef7b7d From 58521d67c5239a3516f9fdb08a35734f46e6ddac Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 452/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 22c6dfefe..8652100df 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 22c6dfefe61703689e7b09185debd3010d93d9db +Subproject commit 8652100dfb9e07f864dff7b9eb3fb144c3640449 From 40f0b93ba154d7da4c0b2c5f565fe89df6bb9f39 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 453/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 9efc29299..61ee5978a 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 9efc29299c878e8b781d7714eebcd5c06cef7b7d +Subproject commit 61ee5978a396f733033af23860014e91969f07e0 From f3cb837c9c2dc10589deaf92c912bccfde093c45 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 9 Jan 2026 18:17:47 -0500 Subject: [PATCH 454/459] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 344a446fc..620b30292 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 344a446fc8588cefe7b812c1fc449a484a27c57b +Subproject commit 620b30292fa572083c77159a1d67f4656b759e1b From 555239dc9cf306ddca0f20cfba69f84de122f431 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 04:56:20 -0300 Subject: [PATCH 455/459] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e8cfc70d7..5e1fe9bca 100644 --- a/README.md +++ b/README.md @@ -664,6 +664,7 @@ Does nothing: [![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) [![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![MPV-Scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=MPV-Scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/MPV-Scripts) ### Spotify From a70079597d44a1b0df2cb585dcc3ff96131b73f3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:38 -0300 Subject: [PATCH 456/459] updated bash-tools --- bash-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash-tools b/bash-tools index 8652100df..11dee29ce 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 8652100dfb9e07f864dff7b9eb3fb144c3640449 +Subproject commit 11dee29cea607445270d8bd675d1bcdecb069c14 From 511b9e6f20cbc66a10f6d4ed88a45311552f5916 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:38 -0300 Subject: [PATCH 457/459] updated pylib --- pylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib b/pylib index 61ee5978a..0d272f4cf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit 61ee5978a396f733033af23860014e91969f07e0 +Subproject commit 0d272f4cf2c27a8ee614eea18c0bd171e47a06cd From 874083a6f3e5ce387d1496b7509c59b30f05aebb Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:39 -0300 Subject: [PATCH 458/459] updated sql --- sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql b/sql index fa8985e60..30d80bbd3 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit fa8985e60aaf5bb26f812cdeeb20b676684e3964 +Subproject commit 30d80bbd33f0e6c26c230f26575cb671c41c3e7d From 32ffee202073282e7b7006102e692c453e98b01c Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 3 Feb 2026 05:04:39 -0300 Subject: [PATCH 459/459] updated templates --- templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates b/templates index 620b30292..f59532dd6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 620b30292fa572083c77159a1d67f4656b759e1b +Subproject commit f59532dd67a583be3522814a15024a561b12c5ec