diff --git a/.appveyor.yml b/.appveyor.yml index d03c2eb3c..740e2f11c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A p p V e y o r C I +# ============================================================================ # + # https://www.appveyor.com/docs/appveyor-yml/ image: Ubuntu @@ -50,7 +54,9 @@ on_finish: # # https://github.com/appveyor/ci/pull/3385 # - #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/install_openssh.sh' | bash -e - + #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/install/install_openssh.sh' | bash -e - + # + # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi install: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 93f047c02..eb0d641fd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u i l d K i t e C I +# ============================================================================ # + # BuildKite Pipeline # # add this command to the UI and it will read the rest of the steps from here: diff --git a/.circleci/config.yml b/.circleci/config.yml index 5dbde36b7..312bf1020 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r c l e C I +# ============================================================================ # + # Master Template with more advanced config: # # https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml @@ -33,12 +37,17 @@ workflows: jobs: build: docker: - - image: cimg/base:2021.04 + - image: cimg/base:2024.12 resource_class: small steps: + # CLI is too old - config validate breaks in test - install new version to fix + # doesn't work - existing version is too old to update + #- run: circleci update + - run: | + curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash - checkout - - setup_remote_docker: - version: 20.10.11 + #- setup_remote_docker: + # version: 20.10.11 - run: setup/ci_bootstrap.sh - run: make init - run: make diff --git a/.cirrus.yml b/.cirrus.yml index d544aeb85..471dc55ba 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -13,14 +13,18 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C i r r u s C I +# ============================================================================ # + # https://cirrus-ci.org/guide/writing-tasks/ container: image: ubuntu:18.04 task: - # doesn't work properly - #skip: "!changesInclude('**/*.md')" + env: + TMPDIR: /var/tmp script: - setup/ci_bootstrap.sh - make init diff --git a/.drone.yml b/.drone.yml index 4d36c9db2..d105ab57a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,6 +15,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# D r o n e C I +# ============================================================================ # + # https://docs.drone.io/quickstart/cli/ # # https://docs.drone.io/cli/install/ diff --git a/.editorconfig b/.editorconfig index 73c19b202..462f53206 100644 --- a/.editorconfig +++ b/.editorconfig @@ -40,7 +40,14 @@ end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true -[*.md] +[{*.md,*.hcl,*.tf,*.tfvars}] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.yml,*.yaml] indent_size = 2 indent_style = space end_of_line = lf diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..e4488d51f --- /dev/null +++ b/.envrc @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# D i r E n v +# ============================================================================ # + +# https://direnv.net/man/direnv-stdlib.1.html + +# See Also: +# +# .envrc-aws +# .envrc-gcp +# .envrc-kubernetes + +# direnv stdlib - loads .envrc from parent dir up to / +# +# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. +# +# bypasses security authorization though - use with care +#source_up +# +# source_up must be loaded before set -u otherwise gets this error: +# +# direnv: loading .envrc +# /bin/bash: line 226: $1: unbound variable +# +# source_up causes this error is up .envrc is found in parent directories: +# +# direnv: No ancestor .envrc found + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +src="$(readlink -f "${BASH_SOURCE[0]}")" +srcdir="$(cd "$(dirname "$src")" && pwd)" + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +# Automatically install Pre-Commit Git hooks if not already present + +if ! type -P pre-commit &>/dev/null; then + if uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Homebrew..." + echo + brew install pre-commit + echo + elif type -P pip &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Pip..." + echo + pip install pre-commit + fi +fi + +if [ -f .pre-commit-config.yaml ] && + type -P pre-commit &>/dev/null && + git rev-parse --is-inside-work-tree &>/dev/null; then + hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" + if [ -L "$hook" ]; then + echo "Detected symlink hook: " + echo + ls -l "$hook" + echo + echo "Removing" + rm -f "$hook" + fi + if ! [ -f "$hook" ]; then + echo + echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." + echo + pre-commit install + fi +fi + +# ============================================================================ # +# D o c k e r C o m p o s e +# ============================================================================ # + +export COMPOSE_PROJECT_NAME="DevOps-Python-tools" + +# ============================================================================ # +# G i t H u b +# ============================================================================ # + +#export GITHUB_ORGANIZATION=HariSekhon + +# ============================================================================ # +# A n s i b l e +# ============================================================================ # + +# use the local repo's ansible.cfg rather than: +# +# $PWD/ansible.cfg +# ~/.ansible.cfg +# /etc/ansible/ansible.cfg +# +# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used +#export ANSIBLE_CONFIG="/path/to/ansible.cfg" + +# ============================================================================ # +# C l o u d f l a r e +# ============================================================================ # + +#export CLOUDFLARE_EMAIL=hari@... +#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens +#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module + +# export the variables for terraform +#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" +#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above + +# ============================================================================ # +# Load External Envrc Files If Present +# ============================================================================ # + +# XXX: safer to bring all these external .envrc inline if you're worried about changes +# to it bypassing 'direnv allow' authorization +load_if_exists(){ + # first arg is a path to a .envrc + # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes + # to pass the context name 'docker-desktop' to switch to + local envrc="$1" + shift + if ! [[ "$envrc" =~ ^/ ]]; then + envrc="$srcdir/$envrc" + fi + if [ -f "$envrc" ]; then + # prevent looping on symlinks to this .envrc if given + if [ "$(readlink "$envrc")" = "$src" ]; then + return + fi + echo + echo "Loading $envrc" + # shellcheck disable=SC1090,SC1091 + . "$envrc" "$@" + fi +} + +# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc +# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses +# to install to system Python) +#load_if_exists ~/.envrc + +# ============================================================================ # +# P y t h o n +# ============================================================================ # + + #.envrc-aws \ + #.envrc-gcp \ + #.envrc-terraform \ +# shellcheck disable=SC2043 +for envrc in \ + .envrc-python \ + ; do + load_if_exists "$envrc" +done + +# ============================================================================ # +# A W S +# ============================================================================ # + +if [[ "$PWD" =~ /aws/ ]]; then + load_if_exists .envrc-aws +fi + +# ============================================================================ # +# G C P +# ============================================================================ # + +if [[ "$PWD" =~ /gcp/ ]]; then + load_if_exists .envrc-gcp +fi + +# ============================================================================ # +# T e r r a f o r m +# ============================================================================ # + +if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then + load_if_exists .envrc-terraform +fi + +# ============================================================================ # +# K u b e r n e t e s +# ============================================================================ # + +if [ -f "$srcdir/.envrc-kubernetes" ]; then + load_if_exists .envrc-kubernetes docker-desktop +fi + +# ============================================================================ # +# . E n v +# ============================================================================ # + +echo +# read .env too +#dotenv + +load_if_exists .envrc.local diff --git a/.envrc-python b/.envrc-python new file mode 100644 index 000000000..6bc2d65b1 --- /dev/null +++ b/.envrc-python @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y t h o n D i r E n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + break + fi +done + +# read .env too +#dotenv diff --git a/.flake8 b/.flake8 index e31fb949f..de51ee708 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,32 @@ +# +# Author: Hari Sekhon +# Date: Mon Oct 21 15:57:10 2019 +0100 +# +# vim:ts=4:sts=4:sw=4:et +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# F l a k e 8 C o n f i g +# ============================================================================ # + +# https://flake8.pycqa.org/en/latest/user/configuration.html + [flake8] -ignore = E265,E402,F401 + max-line-length = 120 + +ignore = E265, + E402, + F401 + exclude = test*/* + max-complexity = 10 diff --git a/.github/workflows/docker_pytools_centos.yaml b/.github/workflows/*_centos.yaml.disabled similarity index 100% rename from .github/workflows/docker_pytools_centos.yaml rename to .github/workflows/*_centos.yaml.disabled diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml index a678d6592..9bed93efb 100644 --- a/.github/workflows/alpine.yaml +++ b/.github/workflows/alpine.yaml @@ -14,7 +14,7 @@ --- name: Alpine -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml index fd88e54cb..f6c4f5ea9 100644 --- a/.github/workflows/alpine_3.yaml +++ b/.github/workflows/alpine_3.yaml @@ -14,7 +14,7 @@ --- name: Alpine 3 -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml.disabled similarity index 97% rename from .github/workflows/centos.yaml rename to .github/workflows/centos.yaml.disabled index ef6aace7b..420b88392 100644 --- a/.github/workflows/centos.yaml +++ b/.github/workflows/centos.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml.disabled similarity index 97% rename from .github/workflows/centos7.yaml rename to .github/workflows/centos7.yaml.disabled index a891a3dbb..ec764d288 100644 --- a/.github/workflows/centos7.yaml +++ b/.github/workflows/centos7.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml.disabled similarity index 97% rename from .github/workflows/centos8.yaml rename to .github/workflows/centos8.yaml.disabled index d39fc2b0a..76c476664 100644 --- a/.github/workflows/centos8.yaml +++ b/.github/workflows/centos8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 8f31b2672..209b3cef1 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -22,7 +22,7 @@ --- name: Checkov -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index 5fdb3401e..168d4914c 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -11,10 +11,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e O w n e r s +# ============================================================================ # + --- -name: Codeowners +name: CodeOwners -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -42,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -52,3 +56,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Validate CODEOWNERS uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml index d56453263..d84cb16b1 100644 --- a/.github/workflows/debian.yaml +++ b/.github/workflows/debian.yaml @@ -14,7 +14,7 @@ --- name: Debian -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml index 735603e28..e857f280f 100644 --- a/.github/workflows/debian_10.yaml +++ b/.github/workflows/debian_10.yaml @@ -14,7 +14,7 @@ --- name: Debian 10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml new file mode 100644 index 000000000..ad765f1ad --- /dev/null +++ b/.github/workflows/debian_11.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 11 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_11.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:11 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml new file mode 100644 index 000000000..dfbc4ec97 --- /dev/null +++ b/.github/workflows/debian_12.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 12 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_12.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:12 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled index 1cb4e5431..f3062b6f8 100644 --- a/.github/workflows/debian_6.yaml.disabled +++ b/.github/workflows/debian_6.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled index e987d7e46..1b0e00295 100644 --- a/.github/workflows/debian_7.yaml.disabled +++ b/.github/workflows/debian_7.yaml.disabled @@ -33,7 +33,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml.disabled similarity index 97% rename from .github/workflows/debian_8.yaml rename to .github/workflows/debian_8.yaml.disabled index 37640f59e..b41ead7d3 100644 --- a/.github/workflows/debian_8.yaml +++ b/.github/workflows/debian_8.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml.disabled similarity index 97% rename from .github/workflows/debian_9.yaml rename to .github/workflows/debian_9.yaml.disabled index 42e28702b..451348091 100644 --- a/.github/workflows/debian_9.yaml +++ b/.github/workflows/debian_9.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml index 769f90e49..7119d6852 100644 --- a/.github/workflows/docker_pytools_alpine.yaml +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Alpine) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml index b2b718ac0..24f8f21dd 100644 --- a/.github/workflows/docker_pytools_debian.yaml +++ b/.github/workflows/docker_pytools_debian.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Debian) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml index 70c545db6..ad1a5ed8d 100644 --- a/.github/workflows/docker_pytools_fedora.yaml +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Fedora) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml index e69018c31..884917919 100644 --- a/.github/workflows/docker_pytools_ubuntu.yaml +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Docker Build (Ubuntu) -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml index 0975be19d..5a4f9a9b4 100644 --- a/.github/workflows/fedora.yaml +++ b/.github/workflows/fedora.yaml @@ -14,7 +14,7 @@ --- name: Fedora -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 4d4ebd577..6b228ca33 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -11,10 +11,16 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k S y n c +# ============================================================================ # + +# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch + --- name: Fork Sync -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: @@ -28,7 +34,7 @@ permissions: contents: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index 00479ecb7..c74fa2a0b 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -11,10 +11,18 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k U p d a t e P R +# ============================================================================ # + +# For a fork of the original repo, activate to keep its branches up to date via Pull Requests +# +# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date + --- name: Fork Update PR -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: @@ -29,7 +37,7 @@ permissions: pull-requests: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml new file mode 100644 index 000000000..1b0fa15b4 --- /dev/null +++ b/.github/workflows/grype.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G r y p e +# ============================================================================ # + +--- +name: Grype + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + Grype: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Grype + uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index fb73746c5..83fa5ef83 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -11,10 +11,16 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# J S O N +# ============================================================================ # + +# Validate any JSON files found in the repo + --- name: JSON -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -48,3 +54,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Check JSON uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml index b73bedf6f..c389f5df0 100644 --- a/.github/workflows/kics.yaml +++ b/.github/workflows/kics.yaml @@ -11,10 +11,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# K i c s +# ============================================================================ # + --- name: Kics -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -42,7 +46,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -52,3 +56,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Kics uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index ac6957e87..438dd184e 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -14,7 +14,7 @@ --- name: Mac -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml index 2d948e136..cc392b105 100644 --- a/.github/workflows/mac_11.yaml +++ b/.github/workflows/mac_11.yaml @@ -14,7 +14,7 @@ --- name: Mac 11 -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml index 7d2bf2dc6..0bcd6e6fd 100644 --- a/.github/workflows/mac_12.yaml +++ b/.github/workflows/mac_12.yaml @@ -14,7 +14,7 @@ --- name: Mac 12 -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml new file mode 100644 index 000000000..72e301ba0 --- /dev/null +++ b/.github/workflows/markdown.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# M a r k D o w n +# ============================================================================ # + +--- +name: Markdown + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + Markdown: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Markdown + uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml.disabled similarity index 95% rename from .github/workflows/pypy2.yaml rename to .github/workflows/pypy2.yaml.disabled index 7dbc25c7f..01080bcc8 100644 --- a/.github/workflows/pypy2.yaml +++ b/.github/workflows/pypy2.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml.disabled similarity index 95% rename from .github/workflows/pypy3.yaml rename to .github/workflows/pypy3.yaml.disabled index ac669813c..101b36751 100644 --- a/.github/workflows/pypy3.yaml +++ b/.github/workflows/pypy3.yaml.disabled @@ -40,7 +40,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml.disabled similarity index 96% rename from .github/workflows/python2.7.yaml rename to .github/workflows/python2.7.yaml.disabled index ff3e51dd1..1813931e7 100644 --- a/.github/workflows/python2.7.yaml +++ b/.github/workflows/python2.7.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml index de6f79440..8ec3008b9 100644 --- a/.github/workflows/python3.10.yaml +++ b/.github/workflows/python3.10.yaml @@ -14,7 +14,7 @@ --- name: Python 3.10 -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml new file mode 100644 index 000000000..0f27b0076 --- /dev/null +++ b/.github/workflows/python3.11.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.11 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.11 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: "3.11" + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml.disabled similarity index 96% rename from .github/workflows/python3.6.yaml rename to .github/workflows/python3.6.yaml.disabled index 84dce85e0..91bd3e673 100644 --- a/.github/workflows/python3.6.yaml +++ b/.github/workflows/python3.6.yaml.disabled @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml index 58ad6c210..73a0045c7 100644 --- a/.github/workflows/python3.7.yaml +++ b/.github/workflows/python3.7.yaml @@ -14,7 +14,7 @@ --- name: Python 3.7 -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml index f1c806dc2..abcbc4c68 100644 --- a/.github/workflows/python3.8.yaml +++ b/.github/workflows/python3.8.yaml @@ -14,7 +14,7 @@ --- name: Python 3.8 -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml index 31c7d0ab8..1ce7eb238 100644 --- a/.github/workflows/python3.9.yaml +++ b/.github/workflows/python3.9.yaml @@ -14,7 +14,7 @@ --- name: Python 3.9 -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 9bb52df1a..c2f614222 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -20,7 +20,7 @@ --- name: Semgrep Cloud -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -58,3 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index cacc83e09..04a3cf197 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -22,7 +22,7 @@ --- name: Semgrep -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -50,7 +50,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -60,3 +60,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 98e53bfe4..246087bd4 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -11,10 +11,16 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S h e l l C h e c k +# ============================================================================ # + +# Validate any shell scripts found in the repo + --- name: ShellCheck -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -40,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 000000000..95b2571ca --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# T r i v y +# ============================================================================ # + +# Scan files in the local repo + +--- +name: Trivy + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + trivy: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Trivy + uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 0d7fd952c..fe863f875 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml.disabled similarity index 97% rename from .github/workflows/ubuntu_14.04.yaml rename to .github/workflows/ubuntu_14.04.yaml.disabled index 395184d7b..1442d675d 100644 --- a/.github/workflows/ubuntu_14.04.yaml +++ b/.github/workflows/ubuntu_14.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml.disabled similarity index 97% rename from .github/workflows/ubuntu_16.04.yaml rename to .github/workflows/ubuntu_16.04.yaml.disabled index 0a55dd214..a9655835d 100644 --- a/.github/workflows/ubuntu_16.04.yaml +++ b/.github/workflows/ubuntu_16.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml.disabled similarity index 97% rename from .github/workflows/ubuntu_18.04.yaml rename to .github/workflows/ubuntu_18.04.yaml.disabled index d909f9529..4b510ba1e 100644 --- a/.github/workflows/ubuntu_18.04.yaml +++ b/.github/workflows/ubuntu_18.04.yaml.disabled @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml index 5145177bb..47d0512ac 100644 --- a/.github/workflows/ubuntu_20.04.yaml +++ b/.github/workflows/ubuntu_20.04.yaml @@ -14,7 +14,7 @@ --- name: Ubuntu 20.04 -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml new file mode 100644 index 000000000..ba8a549b1 --- /dev/null +++ b/.github/workflows/ubuntu_22.04.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 22.04 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_22.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:22.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml index 53307dd73..0037e4ba3 100644 --- a/.github/workflows/ubuntu_github.yaml +++ b/.github/workflows/ubuntu_github.yaml @@ -14,7 +14,7 @@ --- name: GitHub Actions Ubuntu -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -69,7 +69,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 5d924b77b..ab6ab9638 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -11,10 +11,16 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# V a l i d a t i o n +# ============================================================================ # + +# Run all custom validations against files in the repo + --- name: Validation -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -40,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml index f4d3b7035..3f6265a6a 100644 --- a/.github/workflows/xml.yaml +++ b/.github/workflows/xml.yaml @@ -11,10 +11,16 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# X M L +# ============================================================================ # + +# Validate any XML files found in the repo + --- name: XML -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index 65d3ead41..f2a4ca129 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -11,10 +11,16 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# Y A M L +# ============================================================================ # + +# Validate any YAML files found in the repo + --- name: YAML -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -44,7 +50,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -54,3 +60,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Check YAML uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 477b32c3e..625220b32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G i t L a b C I +# ============================================================================ # + # https://docs.gitlab.com/ee/ci/yaml/README.html +#include: '.gitlab/*.y*ml' + image: ubuntu:18.04 job: diff --git a/.mdl.rb b/.mdl.rb new file mode 100644 index 000000000..f8f9b004c --- /dev/null +++ b/.mdl.rb @@ -0,0 +1,30 @@ +#!/usr/bin/env ruby +# vim:ts=4:sts=4:sw=4:et:filetype=ruby +# +# Author: Hari Sekhon +# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +all +#exclude_rule 'MD001' +#exclude_rule 'MD003' +#exclude_rule 'MD005' +exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af +#exclude_rule 'MD012' +exclude_rule 'MD013' # long lines cannot be split if they are URLs +#exclude_rule 'MD022' +#exclude_rule 'MD025' +#exclude_rule 'MD031' +#exclude_rule 'MD032' +exclude_rule 'MD033' # inline HTML is important for formatting +exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines +#exclude_rule 'MD039' +#exclude_rule 'MD056' diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 000000000..27e5b6895 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,5 @@ +mdlrc_dir = File.expand_path('..', __FILE__) + +style_file = File.join(mdlrc_dir, '.mdl.rb') + +style style_file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..787fb79ea --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,74 @@ +# +# Author: Hari Sekhon +# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024) +# +# vim:ts=2:sts=2:sw=2:et +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +--- +fail_fast: false +#exclude: *.tmp$ + +repos: + + # will accept anything that 'git clone' understands + # this means you can set this to a local git repo to develop your own hook repos interactively + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + # Common errors + #- id: end-of-file-fixer # ruins .gitignore Icon\r + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + #- id: forbid-new-submodules + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + + # rewrites python files with useless changes like changing single quotes to double quotes + #- repo: https://github.com/psf/black + # rev: 24.8.0 + # hooks: + # - id: black + + # Git secrets Leaks + - repo: https://github.com/awslabs/git-secrets.git + # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error: + # + # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file + # + rev: 5357e18 + hooks: + - id: git-secrets + + - repo: https://github.com/markdownlint/markdownlint + rev: v0.12.0 + hooks: + - id: markdownlint + name: Markdownlint + description: Run markdownlint on your Markdown files + entry: mdl + args: [-s, .mdl.rb] + language: ruby + files: \.(md|mdown|markdown)$ diff --git a/.pylintrc b/.pylintrc index c20ac8f94..2006619da 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,386 +1,655 @@ -[MASTER] +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y L i n t C o n f i g +# ============================================================================ # + +# pylint --generate-rcfile >> .pylintrc + +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= -# Specify a configuration file. -#rcfile= +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 -# Pickle collected data for later comparisons. -persistent=yes +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 -# List of plugins (as comma separated values of python modules names) to load, +# List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= -# Use multiple processes to speed up Pylint. -jobs=1 +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. -optimize-ast=no +[BASIC] -[MESSAGES CONTROL] +# Naming style matching correct argument names. +argument-naming-style=snake_case -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= +# Naming style matching correct attribute names. +attr-naming-style=snake_case -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111,consider-using-f-string +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata -[REPORTS] +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text +# Naming style matching correct class attribute names. +class-attribute-naming-style=any -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= -# Tells whether to display a full report or only the messages -reports=yes +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= +# Naming style matching correct class names. +class-naming-style=PascalCase +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= -[BASIC] +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,input +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +# Naming style matching correct method names. +method-naming-style=snake_case -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ -# Regular expression matching correct function names -function-rgx=[A-Za-z_][A-Za-z0-9_]{2,30}$ +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Naming style matching correct variable names. +variable-naming-style=snake_case -# Regular expression matching correct constant names -const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ +[CLASSES] -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ +[DESIGN] -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of arguments for function / method. +max-args=5 -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of attributes for a class (see R0902). +max-attributes=7 -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of branch for function / method body. +max-branches=12 -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ +# Maximum number of locals for function / method body. +max-locals=15 -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 +# Maximum number of return / yield for function / method body. +max-returns=6 -[ELIF] +# Maximum number of statements in function / method body. +max-statements=50 -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] -# Maximum number of characters on a single line. -max-line-length=120 +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator -# Maximum number of lines in a module -max-module-lines=1000 +[IMPORTS] -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= [LOGGING] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + # Logging modules to check that the string format arguments are in logging -# function parameter format +# function parameter format. logging-modules=logging +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + missing-class-docstring, + missing-function-docstring, + super-with-arguments, + consider-using-f-string + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO +notes=FIXME, + XXX, + TODO +# Regular expression of note tags to take in consideration. +notes-rgx= -[SIMILARITIES] -# Minimum lines number of a similarity. -min-similarity-lines=4 +[REFACTORING] -# Ignore comments when computing similarities. -ignore-comments=yes +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 -# Ignore docstrings when computing similarities. -ignore-docstrings=yes +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error -# Ignore imports when computing similarities. -ignore-imports=no +[REPORTS] -[SPELLING] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= -# List of comma separated words that should not be checked. -spelling-ignore-words= +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= +# Tells whether to display a full report or only the messages. +reports=no -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no +# Activate the evaluation score. +score=yes -[TYPECHECK] +[SIMILARITIES] -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes +# Comments are removed from the similarity computation +ignore-comments=yes -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= +# Docstrings are removed from the similarity computation +ignore-docstrings=yes -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). This supports can work -# with qualified names. -# -# avoids the following error: -# -# pylint -E ./check_zookeeper_version.py -# -# E: 69,12: Instance of '_socketobject' has no 'sendall' member (no-member) -# E: 70,19: Instance of '_socketobject' has no 'recv' member (no-member) -# -ignored-classes=SQLObject,_socketobject +# Imports are removed from the similarity computation +ignore-imports=yes -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= +# Signatures are removed from the similarity computation +ignore-signatures=yes +# Minimum lines number of a similarity. +min-similarity-lines=4 -[VARIABLES] -# Tells whether we should check for unused import in __init__ files. -init-import=no +[SPELLING] -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: +# List of comma separated words that should not be checked. +spelling-ignore-words= -[CLASSES] +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs +[STRING] -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no -[DESIGN] -# Maximum number of arguments for function / method -max-args=5 +[TYPECHECK] -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager -# Maximum number of locals for function / method body -max-locals=15 +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= -# Maximum number of return / yield for function / method body -max-returns=6 +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes -# Maximum number of branch for function / method body -max-branches=12 +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes -# Maximum number of statements in function / method body -max-statements=50 +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init -# Maximum number of parents for a class (see R0901). -max-parents=7 +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace -# Maximum number of attributes for a class (see R0902). -max-attributes=7 +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin +# List of decorators that change the signature of a decorated function. +signature-mutators= -[IMPORTS] -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec +[VARIABLES] -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= +# List of names allowed to shadow builtins +allowed-redefined-builtins= +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb -[EXCEPTIONS] +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e86c4f36e..59aa80836 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -13,14 +13,19 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# S e m a p h o r e C I +# ============================================================================ # + # https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ version: v1.0 name: DevOps-Python-tools agent: + # https://docs.semaphoreci.com/reference/machine-types#linux machine: type: e1-standard-2 - os_image: ubuntu1804 + os_image: ubuntu2004 execution_time_limit: hours: 3 blocks: @@ -39,6 +44,21 @@ blocks: prologue: commands: - cache restore + # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server + # + # causes error: + # + # Not replacing deleted config file /etc/ssh/sshd_config + # + #- sudo rm -f /etc/ssh/sshd_config + - export DEBIAN_FRONTEND=noninteractive + - sudo -E apt-get update + - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew" + - sudo dpkg --configure -a --force-confmiss --force-confnew + #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections + #- sudo dpkg --configure -a --force-confdef --force-confold + #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + - sudo apt-get install -y openssh-server # each job is separate and could be run on a separate machine so all steps must be together jobs: - name: build @@ -65,9 +85,10 @@ blocks: #- name: DEBUG # value: "1" agent: + # https://docs.semaphoreci.com/reference/machine-types#macos machine: type: a1-standard-4 - os_image: macos-xcode12 + os_image: macos-xcode15 prologue: commands: - cache restore diff --git a/.werckerignore b/.werckerignore deleted file mode 100644 index ff8fb8247..000000000 --- a/.werckerignore +++ /dev/null @@ -1 +0,0 @@ -**/.md diff --git a/Jenkinsfile b/Jenkinsfile index c4ff34fb2..d582bd112 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,3 @@ -#!/usr/bin/env groovy // vim:ts=4:sts=4:sw=4:et:filetype=groovy:syntax=groovy // // Author: Hari Sekhon diff --git a/Makefile b/Makefile index c31c2ce74..1d98ac524 100644 --- a/Makefile +++ b/Makefile @@ -86,12 +86,12 @@ init: # @$(MAKE) $@c %.pyc:: %.py @# this utility script supports taking .pyc or .pyo names and still does the right thing - @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -m py_compile $< && \ echo && \ echo Generated $@ %.pyo:: %.py - @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ python -O -m py_compile $< && \ echo && \ echo Generated $@ @@ -116,20 +116,20 @@ python: pylib fi @# only install pip packages not installed via system packages - @#$(SUDO_PIP) pip install --upgrade -r requirements.txt - @#$(SUDO_PIP) pip install -r requirements.txt - @PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt + @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt + @#$(SUDO_PIP) $(PIP) install -r requirements.txt + @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit - bash-tools/setup/python_install_snakebite.sh || : + PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi - bash-tools/python_pip_install.sh "avro-python3" || : + @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3"; fi + PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3" || : @# for impyla - @#$(SUDO_PIP) pip install --upgrade setuptools || : + @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || : @# @# snappy may fail to install on Mac not finding snappy-c.h - workaround: @# @@ -140,21 +140,21 @@ python: pylib @# /usr/local/include/snappy-c.h @# @# sudo su - @# LD_RUN_PATH=/usr/local/include pip install snappy + @# LD_RUN_PATH=/usr/local/include $(PIP) install snappy @# - @#$(SUDO_PIP) pip install --upgrade -r requirements.txt + @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt @# for ipython-notebook-pyspark.py - @#$(SUDO_PIP) pip install jinja2 + @#$(SUDO_PIP) PIP=$(PIP) install jinja2 @# HiveServer2 - @#$(SUDO_PIP) pip install pyhs2 + @#$(SUDO_PIP) $(PIP) install pyhs2 @# Impala - @#$(SUDO_PIP) pip install impyla + @#$(SUDO_PIP) $(PIP) install impyla @# must downgrade happybase library to work on Python 2.6 - @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi + @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi @# Python >= 2.7 - won't build on 2.6, handle separately and accept failure - @bash-tools/python_pip_install.sh "ipython[notebook]" || : + @PIP=$(PIP) bash-tools/python/python_pip_install.sh "ipython[notebook]" || : @echo $(MAKE) pycompile @echo @@ -165,7 +165,7 @@ python: pylib .PHONY: parquet-tools parquet-tools: - @BIN='.' bash-tools/setup/install_parquet-tools.sh + @BIN='.' bash-tools/install/install_parquet-tools.sh .PHONY: apk-packages-extra apk-packages-extra: @@ -175,7 +175,7 @@ apk-packages-extra: .PHONY: apt-packages-extra apt-packages-extra: - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y default-jdk; fi + if [ -z "$(NOJAVA)" ]; then which java || bash-tools/packages/apt_install_packages.sh default-jdk; fi # for validate_multimedia.py # available in Alpine 2.6, 2.7 and 3.x @@ -189,8 +189,8 @@ apk-packages-multimedia: # Debian 9 Stretch onwards, not available in Debian 8 Jessie .PHONY: apt-packages-multimedia apt-packages-multimedia: - $(SUDO) apt-get update - $(SUDO) apt-get install -y --no-install-recommends ffmpeg + $(SUDO) apt-get update -o DPkg::Lock::Timeout=1200 + $(SUDO) apt-get install -o DPkg::Lock::Timeout=1200 -y --no-install-recommends ffmpeg # for validate_multimedia.py .PHONY: yum-packages-multimedia @@ -200,9 +200,10 @@ yum-packages-multimedia: .PHONY: jython jython: - if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi - if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi - if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + @#if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi + @#if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi + @#if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + bash-tools/packages/install_packages.sh wget expect sh jython_install.sh .PHONY: test-lib @@ -216,7 +217,7 @@ test: .PHONY: basic-test basic-test: test-lib - bash-tools/check_all.sh + bash-tools/checks/check_all.sh .PHONY: install install: build diff --git a/README.md b/README.md index b313668af..5e1fe9bca 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,10 @@ [![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) [![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) +[![LineCount](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2)](https://github.com/boyter/scc/) +[![Cocomo](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2&category=cocomo)](https://github.com/boyter/scc/) [![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-26k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/DevOps-Python-tools) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) [![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) -[![GitLab Pipeline](https://img.shields.io/gitlab/pipeline/harisekhon/DevOps-Python-tools?logo=gitlab&label=GitLab%20CI)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) -[![BitBucket Pipeline](https://img.shields.io/bitbucket/pipelines/harisekhon/devops-python-tools/master?logo=bitbucket&label=BitBucket%20CI)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) -[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml) -[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml) +[![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) +[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) -[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) + [![ShellCheck](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml) [![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) [![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) [![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) +[![Markdown](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml) [![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) +[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) [![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) [![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) -[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) [![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) -[![Docker Build (CentOS)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_centos.yaml) [![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) [![Docker Build (Fedora)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml) [![Docker Build (Ubuntu)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml) @@ -88,54 +87,37 @@ [![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) [![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) [![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) -[![Ubuntu 14.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2014.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+14.04%22) -[![Ubuntu 16.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2016.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+16.04%22) -[![Ubuntu 18.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2018.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22) [![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) +[![Ubuntu 22.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2022.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+22.04%22) [![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) -[![Debian 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+8%22) -[![Debian 9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%209/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22) [![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) -[![CentOS](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22) -[![CentOS 7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%207/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22) -[![CentOS 8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/CentOS%208/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22) +[![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) +[![Debian 12](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2012/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+12%22) [![Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Fedora%22) [![Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine%22) [![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) [![Python versions](https://img.shields.io/badge/Python-2.7+-3776AB?logo=python&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools) -[![Python 2.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%202.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+2.7%22) -[![Python 3.6](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.6/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.6%22) [![Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22) [![Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22) [![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) [![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) -[![PyPy 2](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%202/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22) -[![PyPy 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/PyPy%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22) +[![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) [git.io/pytools](https://git.io/pytools) -### AWS, Docker, Spark, Hadoop, HBase, Hive, Impala, Python & Linux Tools +## AWS, Docker, Spark, Hadoop, HBase, Hive, Impala, Python & Linux Tools DevOps, Cloud, Big Data, NoSQL, Python & Linux tools. All programs have `--help`. -See Also: - - - [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - - [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) - - [DevOps Golang Tools](https://github.com/HariSekhon/DevOps-Golang-tools) - - [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - -repos which contains hundreds more scripts and programs for Cloud, Big Data, SQL, NoSQL, Web and Linux. - Hari Sekhon Cloud & Big Data Contractor, United Kingdom -[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/HariSekhon/) -###### (you're welcome to connect with me on LinkedIn) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) +
*(you're welcome to connect with me on LinkedIn)* -##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### +**Make sure you run `make update` if updating and not just `git pull` as you will often need the latest library submodule and possibly new upstream libraries** ## Quick Start @@ -144,30 +126,37 @@ Cloud & Big Data Contractor, United Kingdom All programs and their pre-compiled dependencies can be found ready to run on [DockerHub](https://hub.docker.com/r/harisekhon/pytools/). List all programs: -``` + +```shell docker run harisekhon/pytools ``` + Run any given program: -``` + +```shell docker run harisekhon/pytools ``` ### Automated Build from source installs git, make, pulls the repo and build the dependencies: -``` + +```shell curl -L https://git.io/python-bootstrap | sh ``` or manually: -``` + +```shell git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools make ``` -To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension instead of `.py`: -``` +To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension +instead of `.py`: + +```shell make anonymize.pyc ``` @@ -177,14 +166,17 @@ Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://git ### Usage -All programs come with a ```--help``` switch which includes a program description and the list of command line options. +All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list +eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and +often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. ### DevOps Python Tools - Inventory - Linux: - - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing + - lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -193,115 +185,201 @@ Environment variables are supported for convenience and also to hide credentials - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` - - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - ```find_active_server.py``` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies - - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) + - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` + - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes + communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support + teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel + before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation + important for anonymization rules, as well as maintaining file content order. On servers this parallelization can + result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file + basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, + useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). + Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See + further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in + your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) - [Amazon Web Services](https://aws.amazon.com/): - - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally + filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have + never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. + Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for + users with passwords not used in the last N days - [Google Cloud Platform](https://cloud.google.com/): - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: - - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) - - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs - - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc. - - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing - - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to + [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is + triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and + [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP + address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its + accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via + VPC connector routing + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with + their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days - [Docker](https://www.docker.com/): - - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a + docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very + useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only + the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests + across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official + `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned + to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy + shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. + `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all + docker images eg. `docker_pull_all_images_all_tags.sh` + - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` - [Spark](https://spark.apache.org/) & Data Format Converters: - - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter - - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas - - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```xml_to_json.py``` - XML to JSON converter - - ```json_to_xml.py``` - JSON to XML converter - - ```json_to_yaml.py``` - JSON to YAML converter - - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - - ```yaml_to_json.py``` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) - - see also ```validate_*.py``` further down for all these formats and more + - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter + - `spark_parquet_to_avro.py` - PySpark Parquet => Avro converter + - `spark_csv_to_avro.py` - PySpark CSV => Avro converter, supports both inferred and explicit schemas + - `spark_csv_to_parquet.py` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas + - `spark_json_to_avro.py` - PySpark JSON => Avro converter + - `spark_json_to_parquet.py` - PySpark JSON => Parquet converter + - `xml_to_json.py` - XML to JSON converter + - `json_to_xml.py` - JSON to XML converter + - `json_to_yaml.py` - JSON to YAML converter + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for + pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and + [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / + directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard + output for convenient command line chaining and redirection, optionally continues on error, collects broken records + to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not + technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) + - see also `validate_*.py` further down for all these formats and more - [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: - [Ambari](https://hortonworks.com/apache/ambari/): - - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API + - `ambari_blueprints.py` - Blueprint cluster templating and deployment tool using Ambari API - list blueprints - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control + - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for + proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool - - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API - - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable + using this tool + - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts + - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API + - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - ```hdfs_find_replication_factor_1.py``` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - ```hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - ```hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to + replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file + or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data + layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster + than doing `hdfs dfs -cat | md5sum`) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree + showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format + to standard output The following programs can all optionally filter by database / table name regex: - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables - - ```hive_tables_row_column_counts.py``` / ```impala_tables_row_column_counts.py``` - outputs tables row and column counts. Useful for finding unusually big tables - - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_metadata.py``` / ```impala_tables_metadata.py``` - outputs for each table the matching regex metadata DDL property from describe table - - ```hive_tables_locations.py``` / ```impala_tables_locations.py``` - outputs for each table its data location + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala + table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation + between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for + finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. + Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where + any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or + subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. + Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain + NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL + property from describe table + - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables - - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, + with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI + tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting + tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, + giving per region row counts and % of total rows for the table as well as median and quartile row counts per + regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in + an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row + counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating + on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an + optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or + memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average + since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / + total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or + average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, + write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular + RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received + the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew + and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase + tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop -- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required - - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig_udfs.jy` - Pig Jython UDFs for Hadoop +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, + useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex + content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for + which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you + want to query cluster wide information available from any online peer (eg. Elasticsearch) + - The following are simplified specialisations of the above program, just pass host arguments, all the details have + been baked in, no switches required + - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA + - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA + - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run + multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server + (run multiple of these for load balancing) + - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node + - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, + Catalog or Statestore + - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator + - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server + - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server + - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node + - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node + - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the + [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI -- ```selenium_hub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - + useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks + session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot + debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as + Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - - ```validate_*.py``` - validate files, directory trees and/or standard input streams + - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: - Avro - CSV @@ -311,21 +389,26 @@ Environment variables are supported for convenience and also to hide credentials - Parquet - XML - YAML - - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) - - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies + - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, + `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) + - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for + things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images + [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & + Linux technologies ### Detailed Build Instructions -##### Python VirtualEnv localized installs - -The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages instead of installing from PyPI then follow the Manual Build section below. +#### Python VirtualEnv localized installs +The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it +detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages +instead of installing from PyPI then follow the Manual Build section below. -#### Manual Setup +### Manual Setup Enter the pytools directory and run git submodule init and git submodule update to fetch my library repo: -``` +```shell git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools git submodule init @@ -333,18 +416,17 @@ git submodule update sudo pip install -r requirements.txt ``` - -#### Offline Setup +### Offline Setup Download the DevOps Python Tools and Pylib git repos as zip files: -https://github.com/HariSekhon/DevOps-Python-tools/archive/master.zip + -https://github.com/HariSekhon/pylib/archive/master.zip + -Unzip both and move Pylib to the ```pylib``` folder under DevOps Python Tools. +Unzip both and move Pylib to the `pylib` folder under DevOps Python Tools. -``` +```shell unzip devops-python-tools-master.zip unzip pylib-master.zip @@ -353,167 +435,246 @@ mv -v pylib-master pylib mv -vf pylib pytools/ ``` -Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). +Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal +mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files. Internal Mirror example ([JFrog Artifactory](https://jfrog.com/artifactory/) or similar): -``` +```shell sudo pip install --index https://host.domain.com/api/pypi/repo/simple --trusted host.domain.com -r requirements.txt ``` Proxy example: -``` +```shell sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt ``` -##### Mac OS X +#### Mac OS X -The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing `git` is enough to trigger Xcode install). +The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing +`git` is enough to trigger Xcode install). -I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): +I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for +development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): -``` -bash-tools/setup/install_homebrew.sh +```shell +bash-tools/install/install_homebrew.sh ``` -``` +```shell brew install openssl wget ``` If failing to build an OpenSSL lib dependency, just prefix the build command like so: -``` +```shell sudo OPENSSL_INCLUDE=/usr/local/opt/openssl/include OPENSSL_LIB=/usr/local/opt/openssl/lib ... ``` -You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is caused by pip 10 vs pip 9 and downgrading will work around it: +You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is +caused by pip 10 vs pip 9 and downgrading will work around it: -``` +```shell sudo pip install --upgrade pip==9.0.1 make sudo pip install --upgrade pip make ``` -### Jython for Hadoop Utils ### +### Jython for Hadoop Utils The 3 Hadoop utility programs listed below require Jython (as well as Hadoop to be installed and correctly configured) -``` +```shell hdfs_time_block_reads.jy hdfs_files_native_checksums.jy hdfs_files_stats.jy ``` Run like so: -``` + +```shell jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` -The ```-J-cp $(hadoop classpath)``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. +The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop +APIs. See below for procedure to install Jython if you don't already have it. -##### Automated Jython Install +#### Automated Jython Install This will download and install jython to /opt/jython-2.7.0: -``` +```shell make jython ``` -##### Manual Jython Install +#### Manual Jython Install -Jython is a simple download and unpack and can be fetched from http://www.jython.org/downloads.html +Jython is a simple download and unpack and can be fetched from Then add the Jython install bin directory to the $PATH or specify the full path to the `jython` binary, eg: -``` +```shell /opt/jython-2.7.0/bin/jython hdfs_time_block_reads.jy ... ``` +### Configuration for Strict Domain / FQDN validation -#### Configuration for Strict Domain / FQDN validation #### - -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my +[PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, +`.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal +environments). -#### Python SSL certificate verification problems +### Python SSL certificate verification problems If you end up with an error like: -``` + +```shell ./dockerhub_show_tags.py centos ubuntu [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:765) ``` -It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One quick fix is to do the following: -``` + +It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One +quick fix is to do the following: + +```shell sudo pip uninstall -y certifi && sudo pip install certifi==2015.04.28 ``` -### Updating ### +### Updating -Run ```make update```. This will git pull and then git submodule update which is necessary to pick up corresponding library updates. +Run: -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run ```make update-no-recompile``` (will miss new library dependencies - do full ```make update``` if you encounter issues). +```shell +make update +``` + +This will git pull and then git submodule update which is necessary to pick up corresponding library updates. + +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each +time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter +issues). ### Testing [Continuous Integration](https://travis-ci.org/HariSekhon/devops-python-tools) is run on this repo with tests for success and failure scenarios: + - unit tests for the custom supporting [python library](https://github.com/HariSekhon/pylib) - integration tests of the top level programs using the libraries for things like option parsing - [functional tests](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/) To trigger all tests run: -``` +```shell make test ``` -which will start with the underlying libraries, then move on to top level integration tests and functional tests using docker containers if docker is available. +which will start with the underlying libraries, then move on to top level integration tests and functional tests using +docker containers if docker is available. -### Contributions ### +### Contributions Patches, improvements and even general feedback are welcome in the form of GitHub pull requests and issue tickets. -## Related Repositories +You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another +Hortonworks guy Jonas Straub: -- [DevOps Bash Tools](https://github.com/HariSekhon/DevOps-Bash-tools) - 800+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies + -- [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery +## Star History -- [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. +[![Star History Chart](https://api.star-history.com/svg?repos=HariSekhon/DevOps-Python-tools&type=Date)](https://star-history.com/#HariSekhon/DevOps-Python-tools&Date) -- [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments +[git.io/python-tools](https://git.io/python-tools) -- [Terraform](https://github.com/HariSekhon/Terraform) - Terraform templates for AWS / GCP / Azure / GitHub management +[git.io/pytools](https://git.io/pytools) -- [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... +## More Core Repos -- [The Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) + -- [HAProxy Configs](https://github.com/HariSekhon/HAProxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. +### Knowledge -- [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3 +[![Knowledge-Base](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Diagrams-as-Code](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) + + + +### DevOps Code + +[![DevOps-Bash-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) +[![DevOps-Python-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Python-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Python-tools) +[![DevOps-Perl-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Perl-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Perl-tools) +[![DevOps-Golang-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Golang-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Golang-tools) -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +### Containerization -- https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb +[![Kubernetes-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Dockerfiles](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) -### Stargazers over time +### CI/CD -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) +[![GitHub-Actions](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Jenkins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) -[git.io/python-tools](https://git.io/python-tools) +### Databases - DBA - SQL -[git.io/pytools](https://git.io/pytools) +[![SQL-scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) + +### DevOps Reloaded + +[![HAProxy-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Terraform](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Packer](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Ansible](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Environments](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) + +### Monitoring + +[![Nagios-Plugins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Nagios-Plugin-Kafka](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Prometheus](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) + +### Templates + +[![Templates](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Template-repo](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) + +### Desktop + +[![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![MPV-Scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=MPV-Scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/MPV-Scripts) + +### Spotify + +[![Spotify-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Spotify-playlists](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) + +The rest of my original source repos are +[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). + +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/) +and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo. + + diff --git a/ambari_blueprints.py b/ambari_blueprints.py index 8e36ba289..e5dd52217 100755 --- a/ambari_blueprints.py +++ b/ambari_blueprints.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/ambari_trigger_service_checks.py b/ambari_trigger_service_checks.py index 8a2247432..2afe64207 100755 --- a/ambari_trigger_service_checks.py +++ b/ambari_trigger_service_checks.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/anonymize.py b/anonymize.py index ee3f023f1..ebcb20ab0 100755 --- a/anonymize.py +++ b/anonymize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # @@ -95,7 +95,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.12' +__version__ = '0.11.0' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -792,46 +792,47 @@ def run(self): # allow to easily switch pre-compilation on/off for testing # testing shows on a moderate sized file that it is a couple secs quicker to use pre-compiled regex def compile(self, name, regex): + log.debug(f"compiling regex '{name}' = '{regex}'") self.regex[name] = re.compile(regex, re.I) #self.regex[name] = regex def prepare_regex(self): self.compile('hostname', - r'(? 2018-01-:00:00 - r'(?!\d+T\d+:\d+)' + \ - r'(?!\d+[^A-Za-z0-9]|' + \ - self.custom_ignores_raw + ')' + \ - '(' + hostname_regex + ')' + \ + r'(?!\d+T\d+:\d+)' + + r'(?!\d+[^A-Za-z0-9]|' + + self.custom_ignores_raw + ')' + + '(' + hostname_regex + ')' + self.negative_host_lookbehind + r':(\d{1,5}(?!\.?\w))', ) self.compile('domain', # don't match java -some.net.property - #r'(?/dev/null || : echo - "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename" + "$srcdir/bash-tools/bin/split.sh" --parts "$parallelism" "$filename" echo "Anonymizing parts" for file_part in "$filename".*; do cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized" diff --git a/aws_s3_presign.py b/aws_s3_presign.py index ec8b6d444..e1addfbbc 100755 --- a/aws_s3_presign.py +++ b/aws_s3_presign.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index 748a248e3..be218c2db 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 903222117..e06a7bcdb 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py index 72c922145..6810abd17 100755 --- a/aws_users_pw_last_used.py +++ b/aws_users_pw_last_used.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py index 58cb30389..befb152ee 100755 --- a/aws_users_unused_access_keys.py +++ b/aws_users_unused_access_keys.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b92c0d211..b430c58dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,24 +13,35 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# A z u r e D e v O p s P i p e l i n e +# ============================================================================ # + # https://aka.ms/yaml trigger: - master +variables: + # ubuntu version + os_version: '22.04' + pool: # there is no /dev/stderr on this azure build! #vmImage: 'ubuntu-latest' - # Ubuntu 16.04 required for docker container support, looks like 18.04 works too - vmImage: 'ubuntu-18.04' + #vmImage: 'ubuntu-22.04' + vmImage: 'ubuntu-$(os_version)' # unprivileged container without sudo, cannot install dependencies -#container: ubuntu:18.04 +#container: ubuntu:22.04 steps: + - script: cat /etc/*-release + displayName: OS Release + # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' - script: env | sort - displayName: env + displayName: Environment # doesn't work in container due to unprivileged execution and lack of sudo #- script: sudo apt-get update && sudo apt-get install -y git make @@ -46,7 +57,7 @@ steps: # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( - script: | - sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c ' + sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c ' set -ex cd /code setup/ci_bootstrap.sh diff --git a/bash-tools b/bash-tools index 1739abe80..11dee29ce 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1739abe80a7bf84d4ca6a7e7834a012473f48159 +Subproject commit 11dee29cea607445270d8bd675d1bcdecb069c14 diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index b63f6a1dc..cdf5243ee 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -13,9 +13,18 @@ # https://www.linkedin.com/in/HariSekhon # -# https://confluence.atlassian.com/x/5Q4SMw -# Only use spaces to indent your .yml configuration. -# ----- +# ============================================================================ # +# B i t b u c k e t C I / C D P i p e l i n e +# ============================================================================ # + +# Reference: +# +# https://support.atlassian.com/bitbucket-cloud/docs/configure-bitbucket-pipelinesyml/ + +# Languages: +# +# https://confluence.atlassian.com/x/5Q4SMw + # You can specify a custom docker image from Docker Hub as your build environment. image: atlassian/default-image:2 diff --git a/buddy.yml b/buddy.yml index 82a6a21ef..e0b57451f 100644 --- a/buddy.yml +++ b/buddy.yml @@ -13,8 +13,13 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# B u d d y C I +# ============================================================================ # + # https://buddy.works/docs/yaml/yaml-schema +--- - pipeline: "Build" trigger_mode: "ON_EVERY_PUSH" ref_name: "master" @@ -22,22 +27,22 @@ target_site_url: "https://github.com/HariSekhon/DevOps-Python-tools" trigger_condition: "ALWAYS" actions: - - action: "Execute: make ci test" - type: "BUILD" - working_directory: "/buddy/devops-python-tools" - docker_image_name: "library/ubuntu" - docker_image_tag: "18.04" - #setup_commands: - # this step gets cached, which results in - # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? - # - apt update - # - apt install -qy git make - execute_commands: - - setup/ci_bootstrap.sh - - make init - - make ci - - make test - volume_mappings: - - "/:/buddy/devops-python-tools" - shell: "BASH" - trigger_condition: "ALWAYS" + - action: "Execute: make ci test" + type: "BUILD" + working_directory: "/buddy/DevOps-Python-tools" + docker_image_name: "library/ubuntu" + docker_image_tag: "18.04" + #setup_commands: + # this step gets cached, which results in + # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? + # - apt update + # - apt install -qy git make + execute_commands: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test + volume_mappings: + - "/:/buddy/DevOps-Python-tools" + shell: "BASH" + trigger_condition: "ALWAYS" diff --git a/center.py b/center.py index 80b17ba0e..379353e68 100755 --- a/center.py +++ b/center.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # args: -s <<< "Auth & Config" # args: -s <<< "GKE Clusters" @@ -44,7 +44,8 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.2' +__version__ = '0.5.1' + class Center(CLI): @@ -56,7 +57,9 @@ def __init__(self): # this doesn't put enough spaces around ampersands, eg. in "Auth & Config" #self.re_bound = re.compile(r'(\b)') self.re_spaces = re.compile(r'(\s)') + self.re_multiple_spaces = re.compile(r'(\s){2}') self.re_chars = re.compile(r'([^\s])(?!\s)') + self.re_chars_spaced = re.compile(r'([^\s])\s') self.timeout_default = None def add_options(self): @@ -66,11 +69,16 @@ def add_options(self): help='No comment prefix handling') self.add_opt('-s', '--space', action='store_true', default=False, help='Space all chars out, makes bigger headings') + self.add_opt('-u', '--unspace', action='store_true', default=False, + help='Removes spaces betweeen chars out, the inverse of --space') def run(self): log_option('width', self.get_opt('width')) log_option('no comment prefix', self.get_opt('no_comment')) log_option('space chars', self.get_opt('space')) + log_option('unspace chars', self.get_opt('unspace')) + if self.get_opt('space') and self.get_opt('unspace'): + self.usage("--space and --unspace are mutually exclusive!") if self.args: self.process_line(' '.join(self.args)) else: @@ -83,6 +91,11 @@ def space(self, line): line = self.re_chars.sub(r'\1 ', line) return line + def unspace(self, line): + line = self.re_chars_spaced.sub(r'\1', line) + line = self.re_multiple_spaces.sub(r'\1', line) + return line + def process_line(self, line): char = '' if not line: @@ -91,7 +104,7 @@ def process_line(self, line): char = ' ' # preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it line = line.strip() - if isChars(line[0], '#'): + if line and isChars(line[0], '#'): char = line[0] line = line.lstrip(char) elif len(line) > 1 and isChars(line[0:1], '/'): @@ -102,9 +115,12 @@ def process_line(self, line): line = line.lstrip(char) if self.get_opt('space'): line = self.space(line) + if self.get_opt('unspace'): + line = self.unspace(line) line = line.strip() side = int(max((self.get_opt('width') - len(line)) / 2, 0)) print(char + ' ' * side + line) + if __name__ == '__main__': Center().main() diff --git a/.concourse.yml b/cicd/.concourse.yml similarity index 80% rename from .concourse.yml rename to cicd/.concourse.yml index 42b806f26..92847121e 100644 --- a/.concourse.yml +++ b/cicd/.concourse.yml @@ -13,6 +13,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o n c o u r s e C I +# ============================================================================ # + +# https://concourse-ci.org/golang-library-example.html + +# https://resource-types.concourse-ci.org/ +# https://concourse-ci.org/resource-types.html resources: - name: github icon: github-circle @@ -25,8 +33,7 @@ resources: # source: # interval: 1d -# https://concourse-ci.org/golang-library-example.html - +# https://concourse-ci.org/jobs.html jobs: - name: build public: false diff --git a/.gocd.yml b/cicd/.gocd.yml similarity index 91% rename from .gocd.yml rename to cicd/.gocd.yml index d3a89a0ef..65583a818 100644 --- a/.gocd.yml +++ b/cicd/.gocd.yml @@ -12,6 +12,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# G o C D +# ============================================================================ # + # https://github.com/tomzo/gocd-yaml-config-plugin#setup # https://docs.gocd.org/current/configuration/configuration_reference.html @@ -19,7 +23,7 @@ --- format_version: 3 pipelines: - devops-python-tools: + DevOps-Python-tools: group: defaultGroup label_template: ${COUNT} lock_behavior: none diff --git a/buildspec.yml b/cicd/buildspec.yml similarity index 100% rename from buildspec.yml rename to cicd/buildspec.yml diff --git a/cloudbuild.yaml b/cicd/cloudbuild.yaml similarity index 100% rename from cloudbuild.yaml rename to cicd/cloudbuild.yaml diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py index 426ad4c4a..bbd94dbb3 100755 --- a/cloudera_navigator_tables_used.py +++ b/cloudera_navigator_tables_used.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py index 324985959..972716236 100755 --- a/cloudera_navigator_tables_used_postgres.py +++ b/cloudera_navigator_tables_used_postgres.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/codefresh.yml b/codefresh.yml index 5326f6e51..7cddcaf7c 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -13,6 +13,10 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e f r e s h C I +# ============================================================================ # + # https://codefresh.io/docs/docs/codefresh-yaml/ version: "1.0" diff --git a/codeship.yml b/codeship.yml deleted file mode 100644 index a65064472..000000000 --- a/codeship.yml +++ /dev/null @@ -1,56 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2021-04-12 18:33:44 +0100 (Mon, 12 Apr 2021) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# ============================================================================ # -# C o d e S h i p -# ============================================================================ # - -# 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this - -# https://github.com/painless-software/codeship-yaml - -# Requires setting up the CodeShip commands like so: -# -# pip install codeship-yaml -# codeship-yaml -# -# or seaparately in sections: -# -# Project Settings > Test Settings > Setup Commands: -# -# pip install codeship-yaml -# codeship-yaml install -# -# Project Settings > Test Settings > Test Commands: -# -# codeship-yaml before_script script -# -# Project Settings > Deployment > (branch name) -# -# codeship-yaml after_success - ---- -install: - # these cause package installation breakages due to GPG or 403 errors, old addresses etc. - - sudo rm -fv -- /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list - - sudo rm -fv -- /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list - - sudo rm -fv -- /etc/apt/sources.list.d/www_apache_org_dist_cassandra_debian.list - - make -#before_script: -# - somecommand -script: - - make test -#after_success: -# - echo "Now we can deploy" diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index ac84f3141..2e675c253 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/docker_registry_show_tags.py b/docker_registry_show_tags.py index c2b29bde1..8c18d3cdc 100755 --- a/docker_registry_show_tags.py +++ b/docker_registry_show_tags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/dockerfiles_check_git_branches.py b/dockerfiles_check_git_branches.py index 444d49ea1..662dce3e2 100755 --- a/dockerfiles_check_git_branches.py +++ b/dockerfiles_check_git_branches.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/dockerfiles_check_git_tags.py b/dockerfiles_check_git_tags.py index 62ec4f361..3e0b627bd 100755 --- a/dockerfiles_check_git_tags.py +++ b/dockerfiles_check_git_tags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/dockerhub_search.py b/dockerhub_search.py index 24f33c692..ba7034145 100755 --- a/dockerhub_search.py +++ b/dockerhub_search.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # args: harisekhon # diff --git a/dockerhub_show_tags.py b/dockerhub_show_tags.py index fafc985f2..59cd6cb5f 100755 --- a/dockerhub_show_tags.py +++ b/dockerhub_show_tags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_apache_drill.py b/find_active_apache_drill.py index ece0d22bf..1e89fb676 100755 --- a/find_active_apache_drill.py +++ b/find_active_apache_drill.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_apache_drill2.py b/find_active_apache_drill2.py index ec1557923..364b0eda3 100755 --- a/find_active_apache_drill2.py +++ b/find_active_apache_drill2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_cassandra.py b/find_active_cassandra.py index 2baa09140..c16ee217c 100755 --- a/find_active_cassandra.py +++ b/find_active_cassandra.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_elasticsearch.py b/find_active_elasticsearch.py index d878f5e42..be449bc79 100755 --- a/find_active_elasticsearch.py +++ b/find_active_elasticsearch.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hadoop2_namenode.py b/find_active_hadoop2_namenode.py index d041b517a..7aad7e77d 100755 --- a/find_active_hadoop2_namenode.py +++ b/find_active_hadoop2_namenode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hadoop_namenode.py b/find_active_hadoop_namenode.py index 0a058fa2b..1f6f2d44d 100755 --- a/find_active_hadoop_namenode.py +++ b/find_active_hadoop_namenode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hadoop_yarn_resource_manager.py b/find_active_hadoop_yarn_resource_manager.py index de6967c4c..f2f0e48e3 100755 --- a/find_active_hadoop_yarn_resource_manager.py +++ b/find_active_hadoop_yarn_resource_manager.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hbase_master.py b/find_active_hbase_master.py index e9edd2a7f..cc7de0222 100755 --- a/find_active_hbase_master.py +++ b/find_active_hbase_master.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hbase_stargate.py b/find_active_hbase_stargate.py index 89a417f84..79dc122fa 100755 --- a/find_active_hbase_stargate.py +++ b/find_active_hbase_stargate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_hbase_thrift.py b/find_active_hbase_thrift.py index 88ea248a7..06e44a414 100755 --- a/find_active_hbase_thrift.py +++ b/find_active_hbase_thrift.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_impala.py b/find_active_impala.py index 4fcc6470a..ae086f46a 100755 --- a/find_active_impala.py +++ b/find_active_impala.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_impala_catalog.py b/find_active_impala_catalog.py index 4b4bdb248..cb6ea0de3 100755 --- a/find_active_impala_catalog.py +++ b/find_active_impala_catalog.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_impala_statestore.py b/find_active_impala_statestore.py index 4edf2c947..049ed78f5 100755 --- a/find_active_impala_statestore.py +++ b/find_active_impala_statestore.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_kubernetes_api.py b/find_active_kubernetes_api.py index 54bbf93d6..a93b0bc81 100755 --- a/find_active_kubernetes_api.py +++ b/find_active_kubernetes_api.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_oozie.py b/find_active_oozie.py index ae23d74fd..156bb51a9 100755 --- a/find_active_oozie.py +++ b/find_active_oozie.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_presto_coordinator.py b/find_active_presto_coordinator.py index 679ec19ae..9b0050a47 100755 --- a/find_active_presto_coordinator.py +++ b/find_active_presto_coordinator.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_server.py b/find_active_server.py index ef2b3e7c2..30810a448 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_active_solrcloud.py b/find_active_solrcloud.py index 0605454b3..659eceb59 100755 --- a/find_active_solrcloud.py +++ b/find_active_solrcloud.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 5735625bd..4c452b03f 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py index d6c07c85b..f982cbddd 100755 --- a/find_missing_files_in_sequence.py +++ b/find_missing_files_in_sequence.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py index 757ebe3f8..f58f86e98 100755 --- a/gcp_service_account_credential_keys.py +++ b/gcp_service_account_credential_keys.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/getent.py b/getent.py index 90f19a4a2..d8b701cd6 100755 --- a/getent.py +++ b/getent.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/git_check_branches_upstream.py b/git_check_branches_upstream.py index f367ab553..27ec09865 100755 --- a/git_check_branches_upstream.py +++ b/git_check_branches_upstream.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_compact_tables.py b/hbase_compact_tables.py index e52db1a69..7bfca40b2 100755 --- a/hbase_compact_tables.py +++ b/hbase_compact_tables.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_flush_tables.py b/hbase_flush_tables.py index 9fd63ec1f..71be25c17 100755 --- a/hbase_flush_tables.py +++ b/hbase_flush_tables.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_generate_data.py b/hbase_generate_data.py index 33e84d38b..3ed31a644 100755 --- a/hbase_generate_data.py +++ b/hbase_generate_data.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_region_requests.py b/hbase_region_requests.py index 24d60bc6e..d182431c8 100755 --- a/hbase_region_requests.py +++ b/hbase_region_requests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_regions_by_memstore_size.py b/hbase_regions_by_memstore_size.py index 14934fca6..295653a81 100755 --- a/hbase_regions_by_memstore_size.py +++ b/hbase_regions_by_memstore_size.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_regions_by_size.py b/hbase_regions_by_size.py index ea4bb5b75..c67184c8b 100755 --- a/hbase_regions_by_size.py +++ b/hbase_regions_by_size.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_regions_least_used.py b/hbase_regions_least_used.py index 8f13bda7c..752be4ed5 100755 --- a/hbase_regions_least_used.py +++ b/hbase_regions_least_used.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_regionserver_requests.py b/hbase_regionserver_requests.py index 7c2638407..a8c96ebb9 100755 --- a/hbase_regionserver_requests.py +++ b/hbase_regionserver_requests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_show_table_region_ranges.py b/hbase_show_table_region_ranges.py index bb2ff7f9c..585e11cf9 100755 --- a/hbase_show_table_region_ranges.py +++ b/hbase_show_table_region_ranges.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_table_region_row_distribution.py b/hbase_table_region_row_distribution.py index 0c7dd702b..0dfe6e024 100755 --- a/hbase_table_region_row_distribution.py +++ b/hbase_table_region_row_distribution.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hbase_table_row_key_distribution.py b/hbase_table_row_key_distribution.py index aeebf0e27..6059763d7 100755 --- a/hbase_table_row_key_distribution.py +++ b/hbase_table_row_key_distribution.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py index 8745848be..1f348556d 100755 --- a/hdfs_find_replication_factor_1.py +++ b/hdfs_find_replication_factor_1.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/headtail.py b/headtail.py index d519608da..ec21a3f5c 100755 --- a/headtail.py +++ b/headtail.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hexanonymize.py b/hexanonymize.py index 847d2d73a..cac35e6a0 100755 --- a/hexanonymize.py +++ b/hexanonymize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py index c1bf4073d..d416f9379 100755 --- a/hive_compute_table_stats.py +++ b/hive_compute_table_stats.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 7fb735421..9d5abf6d6 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index da1d019c4..ea41f3753 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py index 5891882be..d6cb4ce68 100755 --- a/hive_tables_column_counts.py +++ b/hive_tables_column_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_list.py b/hive_tables_list.py index 6b1cc0197..64675c61f 100755 --- a/hive_tables_list.py +++ b/hive_tables_list.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_locations.py b/hive_tables_locations.py index 69f62c0f7..403e07443 100755 --- a/hive_tables_locations.py +++ b/hive_tables_locations.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py index 884715623..d15d62feb 100755 --- a/hive_tables_metadata.py +++ b/hive_tables_metadata.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index 81313be6b..207fcba3f 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 17e85f69a..f1a5752f1 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_row_column_counts.py b/hive_tables_row_column_counts.py index 2185eac7a..c2a5ec7e9 100755 --- a/hive_tables_row_column_counts.py +++ b/hive_tables_row_column_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index b36b46c7c..27aaa1a9c 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 21b592dcf..8ea7e6b75 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py index 95c185b46..04d462e16 100755 --- a/impala_compute_table_stats.py +++ b/impala_compute_table_stats.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 495d77f25..f4e324ac9 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py index 8575c88f9..0ac8491dd 100755 --- a/impala_schemas_csv.py +++ b/impala_schemas_csv.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py index ddfadc37a..e62027542 100755 --- a/impala_tables_column_counts.py +++ b/impala_tables_column_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_list.py b/impala_tables_list.py index b9dd00d5d..a45194ad3 100755 --- a/impala_tables_list.py +++ b/impala_tables_list.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_locations.py b/impala_tables_locations.py index 2203f6caf..383bb1154 100755 --- a/impala_tables_locations.py +++ b/impala_tables_locations.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py index 50ff43fc2..501b82c93 100755 --- a/impala_tables_metadata.py +++ b/impala_tables_metadata.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index 948ba2e8e..2b70caf3f 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py index 2e28d9a4e..4fea8a82e 100755 --- a/impala_tables_null_rows.py +++ b/impala_tables_null_rows.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_row_column_counts.py b/impala_tables_row_column_counts.py index 907d8f321..8cf4d6b82 100755 --- a/impala_tables_row_column_counts.py +++ b/impala_tables_row_column_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 9c968a330..decc2982c 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py index e7fa552d0..d7b49a3b2 100755 --- a/impala_tables_row_counts_any_nulls.py +++ b/impala_tables_row_counts_any_nulls.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/ipython_notebook_pyspark.py b/ipython_notebook_pyspark.py index 7f0c61967..4c4505015 100755 --- a/ipython_notebook_pyspark.py +++ b/ipython_notebook_pyspark.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Author: Hari Sekhon # Date: 6/8/2014 diff --git a/json_docs_to_bulk_multiline.py b/json_docs_to_bulk_multiline.py index 6cb0c21cc..eb49bcbeb 100755 --- a/json_docs_to_bulk_multiline.py +++ b/json_docs_to_bulk_multiline.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/json_to_xml.py b/json_to_xml.py index 0a635c818..54c0d6395 100755 --- a/json_to_xml.py +++ b/json_to_xml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/json_to_yaml.py b/json_to_yaml.py index 172ba2254..674f6729f 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/kics.config b/kics.config new file mode 100644 index 000000000..51ac5712e --- /dev/null +++ b/kics.config @@ -0,0 +1,44 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-05 18:05:53 +0100 (Fri, 05 May 2023) +# +# vim:ts=2:sts=2:sw=2:et:filetype=yaml +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# K i c s C o n f i g +# ============================================================================ # + +# https://github.com/Checkmarx/kics/blob/master/docs/configuration-file.md + +--- +#path: assets/iac_samples +verbose: true +log-file: true +#type: +# - Dockerfile +# - Kubernetes +#queries-path: "assets/queries" +exclude-paths: + # ignore submodules - handle them in the source repos only + - bash-tools/ + - github-actions/ + - haproxy-configs/ + - jenkins/ + - kubernetes-templates/ + - lib/ + - pylib/ + - spotify-tools/ + - sql/ + - sql-keywords/ + - templates/ + - terraform-templates/ +#output-path: "results" diff --git a/opentsdb_import_metric_distribution.py b/opentsdb_import_metric_distribution.py index eb303b545..3b3d3df09 100755 --- a/opentsdb_import_metric_distribution.py +++ b/opentsdb_import_metric_distribution.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py new file mode 100755 index 000000000..a38332443 --- /dev/null +++ b/plot_uk_marriage_rates.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2025-04-26 00:37:28 +0800 (Sat, 26 Apr 2025) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Plots the Marriage rates in England & Wales using the latest download from the UK Government website: + + https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/marriagecohabitationandcivilpartnerships/bulletins/marriagesinenglandandwalesprovisional/2021and2022 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import platform +import subprocess +import sys +#import time +import traceback +import pandas as pd +import matplotlib.pyplot as plt +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +# pylint: disable=too-few-public-methods +class PlotUKMarriageRates(CLI): + + def __init__(self): + # Python 2.x + super(PlotUKMarriageRates, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = 0 + + # def add_options(self): + # super(PlotUKMarriageRates, self).add_options() + # + # def process_options(self): + # super(PlotUKMarriageRates, self).process_options() + + def run(self): + file_path = f"{srcdir}/tests/data/uk_marriage_rates_2022.xslx" + if self.args: + file_path = self.args[0] + #self.usage("Provide path to datadownload.xlsx") + + if not os.path.isfile(file_path): + self.usage(f"Invalid argument provided, not a file: {file_path}") + + log.info(f"Loading file: {file_path}") + xls = pd.ExcelFile(file_path) + + log.info("Parsing xls") + # Read the relevant sheet and skip the metadata + df = xls.parse('Figure 2', skiprows=6) + + # Set proper column headers + df.columns = df.iloc[1] + df = df[2:] # Drop the header rows + + # Rename columns for clarity + df.columns = ['Year', 'Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women'] + + # Convert data types + df['Year'] = df['Year'].astype(int) + for col in ['Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women']: + df[col] = pd.to_numeric(df[col], errors='coerce') + + log.info("Plotting") + plt.figure(figsize=(10, 6)) + plt.plot(df['Year'], df['Opposite-sex Men'], label='Opposite-sex Men') + plt.plot(df['Year'], df['Opposite-sex Women'], label='Opposite-sex Women') + plt.plot(df['Year'], df['Same-sex Men'], label='Same-sex Men') + plt.plot(df['Year'], df['Same-sex Women'], label='Same-sex Women') + + plt.title('Marriage Rates Over Time (England & Wales)') + plt.xlabel('Year') + plt.ylabel('Marriage Rate per 1,000 People') + plt.legend() + plt.grid(True) + plt.tight_layout() + #plt.show() + # + # doesn't show anything because the script finishes before the GUI event loop + # has time to process and display the window + #plt.show(block=False) + # + # pylint: disable=line-too-long + # + # doesn't work even with this hack: + # + # WARNING: NSWindow geometry should only be modified on the main thread! This will raise an exception in the future + # + #import threading + #threading.Thread(target=plt.show).start() + #sleep_secs = 20 + #log.info(f"Sleeping for {sleep_secs} secs to allow you to see the graph pop-up") + #time.sleep(sleep_secs) + + image_path = os.path.splitext(file_path)[0] + '.png' + log.info("Generating output image: {image_path}") + + #if os.path.exists(image_path): + # log.warning(f"Image page already exists, skipping recreating for safety: {image_path}") + # #plt.close() + # return + + # doesn't solve blank png + #plt.gcf().canvas.draw() + + # results in blank png + #plt.savefig(image_path) + + fig = plt.gcf() # draw before saving + fig.canvas.draw() # force rendering + plt.savefig(image_path) + + plt.show() + + if platform.system() == "Darwin": + # fire and forget + # pylint: disable=subprocess-run-check + subprocess.run(['open', image_path]) + + #plt.close() + + +if __name__ == '__main__': + PlotUKMarriageRates().main() diff --git a/pylib b/pylib index be1a86ccb..0d272f4cf 160000 --- a/pylib +++ b/pylib @@ -1 +1 @@ -Subproject commit be1a86ccb881d89aab8a70786c3b5e0e95f36bac +Subproject commit 0d272f4cf2c27a8ee614eea18c0bd171e47a06cd diff --git a/python_find_library_path.py b/python_find_library_path.py index 92417b56e..66fb9c00e 100755 --- a/python_find_library_path.py +++ b/python_find_library_path.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/pythonpath.py b/pythonpath.py index d0dde835c..ef42891b9 100755 --- a/pythonpath.py +++ b/pythonpath.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/quay_show_tags.py b/quay_show_tags.py index 716221ece..d1eef86bf 100755 --- a/quay_show_tags.py +++ b/quay_show_tags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/requirements.txt b/requirements.txt index 6060cbe23..05d204c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ avro==1.8.1 # requires Python == 3.4, build in Makefile instead #avro-python3==1.9.0 -# AWS CLIv1 is obsolete and doesn't support SSO - use CLIv2 - see https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/setup/install_aws_cli.sh +# AWS CLIv1 is obsolete and doesn't support SSO - use CLIv2 - see https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/install/install_aws_cli.sh #awscli==1.16.241 #bitarray==0.8.1 @@ -25,22 +25,32 @@ GitPython==2.1.15 # this GCP API is surprisingly awful, not using #google-api-python-client==1.11.0 -happybase==1.0.0 +# XXX: install broken on new M1 Mac Python 3.9 +#happybase==1.0.0 + humanize==0.5.1 -impyla==0.16.0 +impyla==0.19.0 jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 +matplotlib==3.7.5 + # Python 3.5+ #numpy==1.17.2 -numpy==1.16.5 +# XXX: install broken on new M1 Mac Python 3.9 +#numpy==1.16.5 + +# for plot_uk_marriage_rates.py +openpyxl==3.1.5 +pandas==2.0.3 # requires pg_config to build from source #psycopg2==2.8.4 -psycopg2-binary==2.8.4 +# XXX: install broken on new M1 Mac Python 3.9 +#psycopg2-binary==2.8.4 python-cson==1.0.9 psutil==5.7.0 @@ -60,8 +70,12 @@ psutil==5.7.0 #python-krbV==1.0.90 # needed by avro -python-snappy==0.5 -sasl==0.2.1 +# XXX: install broken on new M1 Mac Python 3.9 +#python-snappy==0.5 + +# XXX: install broken on new M1 Mac Python 3.9 +#sasl==0.2.1 + sh==1.12.14 selenium==3.141.0 @@ -71,9 +85,11 @@ selenium==3.141.0 #snakebite[kerberos]==2.11.0 #snakebite==2.11.0 -thrift-sasl==0.2.1 -thrift==0.9.3 -thriftpy==0.3.9 +# XXX: install broken on new M1 Mac Python 3.9 +#thrift-sasl==0.2.1 +#thrift==0.9.3 +#thriftpy==0.3.9 + toml==0.10.0 xmltodict==0.10.2 yamllint==1.15.0 diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py index f302a3f56..c3d117e53 100755 --- a/selenium_hub_browser_test.py +++ b/selenium_hub_browser_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/serf_event_handler.py b/serf_event_handler.py index 3d1ad7afe..c0d5cacbb 100755 --- a/serf_event_handler.py +++ b/serf_event_handler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.2' +__version__ = '0.2.3' class SerfEventHandler(CLI): @@ -95,7 +95,7 @@ def add_options(self): def enable_commands(self): if self.event in ['query', 'event']: - cmd = None + cmd = '' if self.event == 'query': cmd = self.query_name elif self.event == 'user': diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 94b10159c..d7bedf92a 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -45,11 +45,11 @@ elif [ "$(uname -s)" = Linux ]; then if [ -n "${CI:-}" ]; then export DEBIAN_FRONTEND=noninteractive fi - opts="" + opts="-o DPkg::Lock::Timeout=1200" if [ -z "${PS1:-}" ]; then - opts="-qq" + opts="$opts -qq" fi - $sudo apt-get update $opts + $sudo apt-get update $opts $sudo apt-get install $opts -y git make curl wget --no-install-recommends elif type yum >/dev/null 2>&1; then if grep -qi 'NAME=.*CentOS' /etc/*release; then diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index ab20cfa8d..b01a77d04 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -52,7 +52,7 @@ retry(){ if [ "$(uname -s)" = Darwin ]; then echo "Bootstrapping Mac" # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make) - #retry "$srcdir/install_homebrew.sh" + #retry "$srcdir/../install/install_homebrew.sh" if command -v brew 2>&1; then # fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do @@ -72,8 +72,9 @@ elif [ "$(uname -s)" = Linux ]; then retry $sudo apk update retry $sudo apk add --no-progress bash git make elif type apt-get >/dev/null 2>&1; then - retry $sudo apt-get update -q - retry $sudo apt-get install -qy git make + opts="-q -o DPkg::Lock::Timeout=1200" + retry $sudo apt-get update $opts + retry $sudo apt-get install $opts -y git make elif type yum >/dev/null 2>&1; then #retry $sudo yum makecache retry $sudo yum install -qy git make diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json index 5071aa87d..ce63bf63d 100644 --- a/setup/gocd_config_repo.json +++ b/setup/gocd_config_repo.json @@ -1,5 +1,5 @@ { - "id": "devops-python-tools", + "id": "DevOps-Python-tools", "plugin_id": "yaml.config.plugin", "material": { "type": "git", @@ -12,7 +12,7 @@ "configuration": [ { "key": "file_pattern", - "value": "*.gocd.y*ml" + "value": "cicd/*.gocd.y*ml" } ], "rules": [ diff --git a/shippable.yml b/shippable.yml deleted file mode 100644 index 1fa06bc06..000000000 --- a/shippable.yml +++ /dev/null @@ -1,50 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# http://docs.shippable.com/platform/workflow/config/ - -# http://docs.shippable.com/ci/advancedOptions/environmentVariables/ - -language: none - -branches: - only: - - master - -build: - ci: - # workaround to broken repos - # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB - # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 - # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed. - # DevOps-Python-tools/Makefile.in:272: recipe for target 'apt-packages' failed - - rm -fv -- /etc/apt/sources.list.d/cassandra.sources.list* - - rm -fv -- /etc/apt/sources.list.d/yarn.list* - # Basho repo is giving a '402 payment required' error - # https://github.com/Shippable/support/issues/5172 - - rm -fv -- /etc/apt/sources.list.d/basho_riak.list - #- shippable_retry make - - setup/ci_bootstrap.sh - - make init - - make ci - - make test - -integrations: - notifications: - - integrationName: email - type: email - on_success: never - on_failure: never - on_pull_request: never diff --git a/sonar-project.properties b/sonar-project.properties index 270ec8ac7..319cc6897 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,7 +1,7 @@ # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon -# Date: 2016-07-19 18:18:27 +0100 (Tue, 19 Jul 2016) +# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016) # # https://github.com/HariSekhon/DevOps-Python-tools # @@ -12,21 +12,31 @@ # https://www.linkedin.com/in/HariSekhon # -sonar.projectName=DevOps Python Tools -sonar.projectKey=pytools +# ============================================================================ # +# S o n a r Q u b e +# ============================================================================ # + +sonar.host.url=https://sonarcloud.io + +# Required metadata +sonar.organization=harisekhon +sonar.projectName=DevOps-Python-tools +sonar.projectKey=HariSekhon_DevOps-Python-tools sonar.projectVersion=1.0 -sonar.projectDescription=Python / Jython Tools +sonar.projectDescription=DevOps-Python-tools sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues -sonar.links.ci=https://travis-ci.org/HariSekhon/devops-python-tools +sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions +# directories to scan (defaults to sonar-project.properties dir otherwise) sonar.sources=. #sonar.language=py sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/spark*/**/* +#sonar.exclusions=**/tests/** +sonar.exclusions=**/zookeeper-*/**/* diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py index e1befc15f..8528af1a6 100755 --- a/spark_avro_to_parquet.py +++ b/spark_avro_to_parquet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index 1844e487a..d41717b4f 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -57,6 +57,7 @@ 'com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error @@ -166,7 +167,6 @@ def create_struct(arg): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') @@ -198,5 +198,6 @@ def create_struct(arg): # the databricks avro driver df.write.format('com.databricks.spark.avro').save(avro_dir) + if __name__ == '__main__': SparkCSVToAvro().main() diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py index 90ea7355b..e70f014f6 100755 --- a/spark_csv_to_parquet.py +++ b/spark_csv_to_parquet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -49,6 +49,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-csv_2.11:1.5.0 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error @@ -157,9 +158,7 @@ def create_struct(arg): if not isVersionLax(spark_version): die("Spark version couldn't be determined. " + support_msg('pytools')) - # pylint: disable=invalid-name - - df = None + # pylint: disable=invalid-name if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index 6dab95cc5..db9d069ed 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error @@ -109,7 +110,6 @@ def run(self): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): df = sqlContext.read.json(json_file) else: diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py index f6fa7e7bf..6976ce153 100755 --- a/spark_json_to_parquet.py +++ b/spark_json_to_parquet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -43,6 +43,7 @@ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) sys.exit(4) pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py index cd026dd19..030816963 100755 --- a/spark_parquet_to_avro.py +++ b/spark_parquet_to_avro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error diff --git a/sql b/sql index fa0a59ca9..30d80bbd3 160000 --- a/sql +++ b/sql @@ -1 +1 @@ -Subproject commit fa0a59ca990748408802732d50202522b82d52c7 +Subproject commit 30d80bbd33f0e6c26c230f26575cb671c41c3e7d diff --git a/strip_ansi_escape_codes.py b/strip_ansi_escape_codes.py index 021f59b00..24738194a 100755 --- a/strip_ansi_escape_codes.py +++ b/strip_ansi_escape_codes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/.teamcity.vcs.json b/teamcity/.teamcity.vcs.json similarity index 100% rename from .teamcity.vcs.json rename to teamcity/.teamcity.vcs.json diff --git a/.teamcity.vcs.oauth.json b/teamcity/.teamcity.vcs.oauth.json similarity index 100% rename from .teamcity.vcs.oauth.json rename to teamcity/.teamcity.vcs.oauth.json diff --git a/.teamcity.vcs.ssh.json b/teamcity/.teamcity.vcs.ssh.json similarity index 100% rename from .teamcity.vcs.ssh.json rename to teamcity/.teamcity.vcs.ssh.json diff --git a/templates b/templates index 27c38c908..f59532dd6 160000 --- a/templates +++ b/templates @@ -1 +1 @@ -Subproject commit 27c38c908387ceacc51463578326ba996449da30 +Subproject commit f59532dd67a583be3522814a15024a561b12c5ec diff --git a/tests/all.sh b/tests/all.sh index 33452eb7c..dbac7b8fb 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -31,7 +31,7 @@ section "Running PyTools ALL" cd "$srcdir/.."; # shellcheck disable=SC1090 # has to be included so that isExcluded function is inherited -. "$srcdir/../bash-tools/check_all.sh" +. "$srcdir/../bash-tools/checks/check_all.sh" #tests/test_yamllint.sh @@ -41,4 +41,4 @@ exit 0 # pyspark not found tests/help.sh -bash-tools/run_tests.sh +bash-tools/checks/run_tests.sh diff --git a/tests/data/uk_marriage_rates_2022.xslx b/tests/data/uk_marriage_rates_2022.xslx new file mode 100644 index 000000000..54efc3321 Binary files /dev/null and b/tests/data/uk_marriage_rates_2022.xslx differ diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index 92e3d2e9d..2a4f9c48e 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -36,7 +36,7 @@ check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED # shellcheck disable=SC2016 -check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" +check '[ "$(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep -c "^harisekhon/[A-Za-z0-9_-]*$")" = 40 ]' "DockerHub Search quiet mode for shell scripting" echo echo diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index c86449aac..3493bae6d 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -32,7 +32,7 @@ if ! type -P ffmpeg &>/dev/null; then echo "WARNING: ffmpeg not installed, skipping validate_multimedia.py tests" exit 0 if type -P apt-get &>/dev/null; then - sudo apt-get install -y ffmpeg + sudo apt-get install -o DPkg::Lock::Timeout=1200 -y ffmpeg elif type -P yum &>/dev/null; then echo "WARNING: cannot auto-install ffmpeg on RHEL/CentOS, the 3rd party repos and deps are seriously broken" fi diff --git a/timeout.py b/timeout.py index 6e33334ec..831d2e99f 100755 --- a/timeout.py +++ b/timeout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/travis_debug_session.py b/travis_debug_session.py index 1c487b221..aafebf35e 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/travis_last_log.py b/travis_last_log.py index 2aa861cff..82b7493e5 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/urldecode.py b/urldecode.py index 7356b84db..87319d3be 100755 --- a/urldecode.py +++ b/urldecode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/urlencode.py b/urlencode.py index f494f0d3f..6ca907f15 100755 --- a/urlencode.py +++ b/urlencode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_avro.py b/validate_avro.py index 718d05690..37eaab848 100755 --- a/validate_avro.py +++ b/validate_avro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_cson.py b/validate_cson.py index a64ccbc78..6e017ab97 100755 --- a/validate_cson.py +++ b/validate_cson.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_csv.py b/validate_csv.py index 2e7d2f78c..bac1d583e 100755 --- a/validate_csv.py +++ b/validate_csv.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_ini.py b/validate_ini.py index 6d731c864..b38b3ab6a 100755 --- a/validate_ini.py +++ b/validate_ini.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.12.2' +__version__ = '0.12.3' class IniValidatorTool(CLI): @@ -72,7 +72,7 @@ def __init__(self): self.re_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I) # In Windows ini key cannot contain equals sign = or semicolon ; # key=val or [section] - self.re_ini_section = re.compile(r'^\s*\[([\w=\:\.-]+)\]\s*$') + self.re_ini_section = re.compile(r'^\s*\[([\w\s=\:\.-]+)\]\s*$') self.re_ini_key = re.compile(r'^\s*(?:[^\[;=]+)s*$') # INI value can be anything .* so not regex'ing it self.valid_ini_msg = ' => INI OK' diff --git a/validate_ini2.py b/validate_ini2.py index a987a9474..5c41436a4 100755 --- a/validate_ini2.py +++ b/validate_ini2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_json.py b/validate_json.py index 3ffa53821..dbca2c431 100755 --- a/validate_json.py +++ b/validate_json.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_ldap_ldif.py b/validate_ldap_ldif.py index d27f333d7..963b725c8 100755 --- a/validate_ldap_ldif.py +++ b/validate_ldap_ldif.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # diff --git a/validate_multimedia.py b/validate_multimedia.py index 635833c3f..0362bb30d 100755 --- a/validate_multimedia.py +++ b/validate_multimedia.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_parquet.py b/validate_parquet.py index c707d6c90..bf6690d06 100755 --- a/validate_parquet.py +++ b/validate_parquet.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_toml.py b/validate_toml.py index 96e029619..25628d222 100755 --- a/validate_toml.py +++ b/validate_toml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_xml.py b/validate_xml.py index 10b73b438..c954bac42 100755 --- a/validate_xml.py +++ b/validate_xml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/validate_yaml.py b/validate_yaml.py index ebfb11ef8..75ee4034c 100755 --- a/validate_yaml.py +++ b/validate_yaml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/welcome.py b/welcome.py index d9246e6b4..d9c1b2546 100755 --- a/welcome.py +++ b/welcome.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon diff --git a/wercker.yml b/wercker.yml deleted file mode 100644 index c27be30ed..000000000 --- a/wercker.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Author: Hari Sekhon -# Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020) -# -# vim:ts=2:sts=2:sw=2:et -# -# https://github.com/HariSekhon/DevOps-Python-tools -# -# License: see accompanying Hari Sekhon LICENSE file -# -# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -# -# https://www.linkedin.com/in/HariSekhon -# - -# https://devcenter.wercker.com/reference/wercker-yml/ - -box: debian - -build: - steps: - - script: - name: ci bootstrap - code: setup/ci_bootstrap.sh - - script: - name: init - code: make init - - script: - name: build - code: make ci - - script: - name: test - code: make test diff --git a/xml_to_json.py b/xml_to_json.py index bd16d8c46..14d4cde62 100755 --- a/xml_to_json.py +++ b/xml_to_json.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/xml_to_yaml.py b/xml_to_yaml.py index 44ad5608f..47a098243 100755 --- a/xml_to_yaml.py +++ b/xml_to_yaml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon @@ -11,7 +11,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/yaml_to_json.py b/yaml_to_json.py index 4fa3432cc..90a5db9b7 100755 --- a/yaml_to_json.py +++ b/yaml_to_json.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # args: ../bash-tools/.gitlab-ci.yml # @@ -12,7 +12,7 @@ # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """