diff --git a/.appveyor.yml b/.appveyor.yml
new file mode 100644
index 000000000..740e2f11c
--- /dev/null
+++ b/.appveyor.yml
@@ -0,0 +1,89 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# A p p V e y o r C I
+# ============================================================================ #
+
+# https://www.appveyor.com/docs/appveyor-yml/
+
+image: Ubuntu
+
+# workaround for default JDK9 have old CA certs:
+#
+# https://github.com/appveyor/ci/issues/3833
+#
+# https://www.appveyor.com/docs/getting-started-with-appveyor-for-linux/#configuring-language-stack
+#
+stack: jdk 15
+
+skip_commits:
+ files:
+ - docs/*
+ - '**/*.md'
+
+# https://www.appveyor.com/docs/how-to/ssh-to-build-worker/
+environment:
+ APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension
+
+# enable SSH session accessible via my public key
+#init:
+# - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -
+
+# more useful at end to leverage .appveyor.yml tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to do all that manually in SSH session
+on_finish:
+ # set this in Settings -> Environment dynamically instead of here
+ #- sh: export APPVEYOR_SSH_BLOCK=true
+ #
+ # workaround for https://github.com/appveyor/ci/issues/3373
+ # and https://github.com/appveyor/ci/issues/3384
+ #
+ # has since been added to AppVeyor's own scripts:
+ #
+ # https://github.com/appveyor/ci/pull/3385
+ #
+ #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/install/install_openssh.sh' | bash -e -
+ #
+ # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/
+ - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi
+
+install:
+ # workaround for:
+ # Some packages could not be installed. This may mean that you have
+ # requested an impossible situation or if you are using the unstable
+ # distribution that some required packages have not yet been created
+ # or been moved out of Incoming.
+ # The following information may help to resolve the situation:
+ #
+ # The following packages have unmet dependencies:
+ # mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed
+ # E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages.
+ # DevOps-Python-tools/Makefile.in:272: recipe for target 'apt-packages' failed
+ # make[2]: *** [apt-packages] Error 123
+ # make[2]: Leaving directory '/home/appveyor/projects/pylib'
+ # DevOps-Python-tools/Makefile.in:212: recipe for target 'system-packages' failed
+ #
+ # adding "|| :" to the end of these commands causes them to be silently ignored!
+ - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list
+ - sudo apt purge -yq --allow-change-held-packages mssql-server
+ # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411
+ #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages
+ - setup/ci_bootstrap.sh
+ - make
+
+test_script:
+ - make test
+
+build: off
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
new file mode 100644
index 000000000..eb0d641fd
--- /dev/null
+++ b/.buildkite/pipeline.yml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-03-13 21:10:39 +0000 (Fri, 13 Mar 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# B u i l d K i t e C I
+# ============================================================================ #
+
+# BuildKite Pipeline
+#
+# add this command to the UI and it will read the rest of the steps from here:
+#
+# - command: buildkite-agent pipeline upload
+
+# Yaml Anchors workaround to BuildKite's lack of global retries configuration - credit to Jason @ BuildKite for this workaround:
+ #
+# https://forum.buildkite.community/t/reschedule-builds-on-other-agents-rather-than-fail-builds-when-agents-time-out-or-are-killed-machine-shut-down-or-put-to-sleep/1388/5
+#
+anchors:
+ std_retries: &std_retries
+ retry:
+ automatic:
+ - exit_status: -1 # Agent was lost
+ limit: 2
+ - exit_status: 255 # Forced agent shutdown
+ limit: 2
+
+steps:
+ - command: setup/ci_bootstrap.sh
+ label: ci bootstrap
+ timeout: 30 # brew can take 10 mins just to do a brew update
+ branches: master
+ <<: [*std_retries]
+ - wait
+ - command: make init
+ label: init
+ timeout: 2
+ branches: master
+ <<: [*std_retries]
+ - wait
+ - command: make ci
+ label: build
+ timeout: 60
+ branches: master
+ <<: [*std_retries]
+ - wait
+ - command: make test
+ label: test
+ timeout: 120
+ branches: master
+ <<: [*std_retries]
diff --git a/.checkov.yaml b/.checkov.yaml
new file mode 100644
index 000000000..8962e2283
--- /dev/null
+++ b/.checkov.yaml
@@ -0,0 +1,48 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-21 16:53:29 +0000 (Mon, 21 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C h e c k o v c o n f i g
+# ============================================================================ #
+
+# https://github.com/bridgecrewio/checkov#configuration-using-a-config-file
+#
+# This is not well documented but the fields seem to be the same as:
+#
+# checkov --help
+#
+# See master template at:
+#
+# https://github.com/HariSekhon/Templates/blob/master/.checkov.yaml
+
+---
+compact: true
+directory:
+ - .
+skip-path:
+ - bash-tools
+ - pylib
+ - sql
+ - templates
+docker-image: harisekhon/pytools
+download-external-modules: true # without this gets lots of annoying warning lines such as '2022-02-22 16:14:40,180 [MainThread ] [WARNI] Failed to download module x/y/z:n.n.n'
+framework:
+ - all
+no-guide: true
+output: cli
+quiet: true
+repo-id: HariSekhon/DevOps-Python-tools # what to report to Bridgecrew Cloud - without this gets annoying duplicate repos such as 'harisekhon_cli_repo/pytools'
+skip-suppressions: true
+soft-fail: true
diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 000000000..312bf1020
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,54 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-23 23:30:14 +0000 (Sun, 23 Feb 2020)
+# Original: H1 2016 (Circle CI 1.x)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C i r c l e C I
+# ============================================================================ #
+
+# Master Template with more advanced config:
+#
+# https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml
+
+# Reference:
+#
+# https://circleci.com/docs/2.0/configuration-reference
+
+version: 2.1
+
+workflows:
+ version: 2
+ workflow:
+ jobs:
+ - build
+
+jobs:
+ build:
+ docker:
+ - image: cimg/base:2024.12
+ resource_class: small
+ steps:
+ # CLI is too old - config validate breaks in test - install new version to fix
+ # doesn't work - existing version is too old to update
+ #- run: circleci update
+ - run: |
+ curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash
+ - checkout
+ #- setup_remote_docker:
+ # version: 20.10.11
+ - run: setup/ci_bootstrap.sh
+ - run: make init
+ - run: make
+ - run: make test
diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 000000000..471dc55ba
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,31 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-24 16:55:36 +0000 (Mon, 24 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C i r r u s C I
+# ============================================================================ #
+
+# https://cirrus-ci.org/guide/writing-tasks/
+
+container:
+ image: ubuntu:18.04
+
+task:
+ env:
+ TMPDIR: /var/tmp
+ script:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci test
diff --git a/.drone.yml b/.drone.yml
new file mode 100644
index 000000000..d105ab57a
--- /dev/null
+++ b/.drone.yml
@@ -0,0 +1,49 @@
+---
+# XXX: putting this separator further down with code causes a parsing bug in drone lint
+#
+# Author: Hari Sekhon
+# Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# D r o n e C I
+# ============================================================================ #
+
+# https://docs.drone.io/quickstart/cli/
+#
+# https://docs.drone.io/cli/install/
+#
+# brew install drone-cli
+#
+# cd to this directory
+#
+# drone exec [--pipeline default] [--include=thisstep] [--exclude=thatstep]
+
+kind: pipeline
+type: docker
+name: default
+
+steps:
+ - name: build
+ image: ubuntu:18.04
+ #environment:
+ # DEBUG: 1
+ commands:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
+
+trigger:
+ branch:
+ - master
diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 000000000..462f53206
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,85 @@
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# http://EditorConfig.org
+
+# stop recursing upwards for other .editorconfig files
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+indent_size = 4
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.go]
+indent_size = 4
+indent_style = tab
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[Makefile]
+indent_size = 4
+indent_style = tab
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[{*.md,*.hcl,*.tf,*.tfvars}]
+indent_size = 2
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.yml,*.yaml]
+indent_size = 2
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[.*]
+indent_size = 4
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+# ============================================================================ #
+# Older Stuff, don't think I use this any more
+# ============================================================================ #
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+#[*.{js,py}]
+#charset = utf-8
+
+# Indentation override for all JS under lib directory
+#[lib/**.js]
+#indent_style = space
+#indent_size = 2
+
+# Matches the exact files either package.json or .travis.yml
+#[{package.json,.travis.yml}]
+#indent_style = space
+#indent_size = 2
+
+#[*.xml]
+#indent_style = space
+#indent_size = 2
diff --git a/.envrc b/.envrc
new file mode 100644
index 000000000..e4488d51f
--- /dev/null
+++ b/.envrc
@@ -0,0 +1,214 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: Mon Feb 22 17:42:01 2021 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# D i r E n v
+# ============================================================================ #
+
+# https://direnv.net/man/direnv-stdlib.1.html
+
+# See Also:
+#
+# .envrc-aws
+# .envrc-gcp
+# .envrc-kubernetes
+
+# direnv stdlib - loads .envrc from parent dir up to /
+#
+# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc.
+#
+# bypasses security authorization though - use with care
+#source_up
+#
+# source_up must be loaded before set -u otherwise gets this error:
+#
+# direnv: loading .envrc
+# /bin/bash: line 226: $1: unbound variable
+#
+# source_up causes this error is up .envrc is found in parent directories:
+#
+# direnv: No ancestor .envrc found
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+src="$(readlink -f "${BASH_SOURCE[0]}")"
+srcdir="$(cd "$(dirname "$src")" && pwd)"
+
+# ============================================================================ #
+# P r e - C o m m i t
+# ============================================================================ #
+
+# Automatically install Pre-Commit Git hooks if not already present
+
+if ! type -P pre-commit &>/dev/null; then
+ if uname -s | grep -q Darwin &&
+ type -P brew &>/dev/null; then
+ echo
+ echo "Pre-commit is not installed - installing now using Homebrew..."
+ echo
+ brew install pre-commit
+ echo
+ elif type -P pip &>/dev/null; then
+ echo
+ echo "Pre-commit is not installed - installing now using Pip..."
+ echo
+ pip install pre-commit
+ fi
+fi
+
+if [ -f .pre-commit-config.yaml ] &&
+ type -P pre-commit &>/dev/null &&
+ git rev-parse --is-inside-work-tree &>/dev/null; then
+ hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
+ if [ -L "$hook" ]; then
+ echo "Detected symlink hook: "
+ echo
+ ls -l "$hook"
+ echo
+ echo "Removing"
+ rm -f "$hook"
+ fi
+ if ! [ -f "$hook" ]; then
+ echo
+ echo "Pre-commit hook is not installed in local Git repo checkout - installing now..."
+ echo
+ pre-commit install
+ fi
+fi
+
+# ============================================================================ #
+# D o c k e r C o m p o s e
+# ============================================================================ #
+
+export COMPOSE_PROJECT_NAME="DevOps-Python-tools"
+
+# ============================================================================ #
+# G i t H u b
+# ============================================================================ #
+
+#export GITHUB_ORGANIZATION=HariSekhon
+
+# ============================================================================ #
+# A n s i b l e
+# ============================================================================ #
+
+# use the local repo's ansible.cfg rather than:
+#
+# $PWD/ansible.cfg
+# ~/.ansible.cfg
+# /etc/ansible/ansible.cfg
+#
+# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used
+#export ANSIBLE_CONFIG="/path/to/ansible.cfg"
+
+# ============================================================================ #
+# C l o u d f l a r e
+# ============================================================================ #
+
+#export CLOUDFLARE_EMAIL=hari@...
+#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens
+#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module
+
+# export the variables for terraform
+#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL"
+#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above
+
+# ============================================================================ #
+# Load External Envrc Files If Present
+# ============================================================================ #
+
+# XXX: safer to bring all these external .envrc inline if you're worried about changes
+# to it bypassing 'direnv allow' authorization
+load_if_exists(){
+ # first arg is a path to a .envrc
+ # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes
+ # to pass the context name 'docker-desktop' to switch to
+ local envrc="$1"
+ shift
+ if ! [[ "$envrc" =~ ^/ ]]; then
+ envrc="$srcdir/$envrc"
+ fi
+ if [ -f "$envrc" ]; then
+ # prevent looping on symlinks to this .envrc if given
+ if [ "$(readlink "$envrc")" = "$src" ]; then
+ return
+ fi
+ echo
+ echo "Loading $envrc"
+ # shellcheck disable=SC1090,SC1091
+ . "$envrc" "$@"
+ fi
+}
+
+# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc
+# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses
+# to install to system Python)
+#load_if_exists ~/.envrc
+
+# ============================================================================ #
+# P y t h o n
+# ============================================================================ #
+
+ #.envrc-aws \
+ #.envrc-gcp \
+ #.envrc-terraform \
+# shellcheck disable=SC2043
+for envrc in \
+ .envrc-python \
+ ; do
+ load_if_exists "$envrc"
+done
+
+# ============================================================================ #
+# A W S
+# ============================================================================ #
+
+if [[ "$PWD" =~ /aws/ ]]; then
+ load_if_exists .envrc-aws
+fi
+
+# ============================================================================ #
+# G C P
+# ============================================================================ #
+
+if [[ "$PWD" =~ /gcp/ ]]; then
+ load_if_exists .envrc-gcp
+fi
+
+# ============================================================================ #
+# T e r r a f o r m
+# ============================================================================ #
+
+if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then
+ load_if_exists .envrc-terraform
+fi
+
+# ============================================================================ #
+# K u b e r n e t e s
+# ============================================================================ #
+
+if [ -f "$srcdir/.envrc-kubernetes" ]; then
+ load_if_exists .envrc-kubernetes docker-desktop
+fi
+
+# ============================================================================ #
+# . E n v
+# ============================================================================ #
+
+echo
+# read .env too
+#dotenv
+
+load_if_exists .envrc.local
diff --git a/.envrc-python b/.envrc-python
new file mode 100644
index 000000000..6bc2d65b1
--- /dev/null
+++ b/.envrc-python
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: Mon Feb 22 17:42:01 2021 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# P y t h o n D i r E n v
+# ============================================================================ #
+
+# .envrc to auto-load the virtualenv inside the 'venv' directory if present
+
+# https://direnv.net/man/direnv-stdlib.1.html
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default
+for venv in "$PWD/venv" "$HOME/venv"; do
+ if [ -f "$venv/bin/activate" ]; then
+ echo
+ echo "Virtualenv directory found in: $venv"
+ echo
+ echo "Activating Virtualenv inside the directory: $venv"
+
+ # shellcheck disable=SC1091
+ source "$venv/bin/activate"
+ break
+ fi
+done
+
+# read .env too
+#dotenv
diff --git a/.flake8 b/.flake8
new file mode 100644
index 000000000..de51ee708
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,32 @@
+#
+# Author: Hari Sekhon
+# Date: Mon Oct 21 15:57:10 2019 +0100
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# F l a k e 8 C o n f i g
+# ============================================================================ #
+
+# https://flake8.pycqa.org/en/latest/user/configuration.html
+
+[flake8]
+
+max-line-length = 120
+
+ignore = E265,
+ E402,
+ F401
+
+exclude = test*/*
+
+max-complexity = 10
diff --git a/.gitallowed b/.gitallowed
new file mode 100644
index 000000000..bb1dbfb76
--- /dev/null
+++ b/.gitallowed
@@ -0,0 +1,9 @@
+AKIAIOSFODNN7EXAMPLE
+
+wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
+AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE
+
+ASIAIOSFODNN7EXAMPLE
+
+AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000..ad861088b
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,38 @@
+#
+# Author: Hari Sekhon
+# Date: 2021-11-09 15:14:59 +0000 (Tue, 09 Nov 2021)
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+
+# Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam
+
+# Tips:
+#
+# * includes changes under .github/
+# dir/* only matches first level file changes but doesn't recurse
+# dir/ recurses
+#
+# - CODEOWNERS in base branch of PR determines review request
+# - paths are case sensitive
+# - last match wins, use * at top for overall owner then override with more specific teams
+
+#* @harisekhon # username or email address
+#* @myorg/platform-engineering # team based is the way to go - team must have Write access to the repo regardless of if individuals have access
+#* @myorg/devops
+#k8s @myorg/devops @myorg/sre-team
+#apps/ @myorg/developers
+#apps/dir2 # ignores dir2 as no owner/team specified on this line
+#src/ @myorg/developers
+#docs/ docs@example.com
+#.github/workflows @ci-cd-team
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
new file mode 100644
index 000000000..aa46b9847
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE.md
@@ -0,0 +1,3 @@
+Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell.
+
+You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) respectively.
diff --git a/.github/workflows/*_centos.yaml.disabled b/.github/workflows/*_centos.yaml.disabled
new file mode 100644
index 000000000..51d70819e
--- /dev/null
+++ b/.github/workflows/*_centos.yaml.disabled
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (CentOS)
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:centos
+ ghcr.io/harisekhon/pytools:centos
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-centos
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml
new file mode 100644
index 000000000..9bed93efb
--- /dev/null
+++ b/.github/workflows/alpine.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Alpine
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: alpine:latest
+ caches: apk pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml
new file mode 100644
index 000000000..f6c4f5ea9
--- /dev/null
+++ b/.github/workflows/alpine_3.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Alpine 3
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine_3.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: alpine:3
+ caches: apk pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/centos.yaml.disabled b/.github/workflows/centos.yaml.disabled
new file mode 100644
index 000000000..420b88392
--- /dev/null
+++ b/.github/workflows/centos.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: CentOS
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/centos.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: centos:latest
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/centos7.yaml.disabled b/.github/workflows/centos7.yaml.disabled
new file mode 100644
index 000000000..ec764d288
--- /dev/null
+++ b/.github/workflows/centos7.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: CentOS 7
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/centos7.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: centos:7
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/centos8.yaml.disabled b/.github/workflows/centos8.yaml.disabled
new file mode 100644
index 000000000..76c476664
--- /dev/null
+++ b/.github/workflows/centos8.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: CentOS 8
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/centos8.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: centos:8
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml
new file mode 100644
index 000000000..209b3cef1
--- /dev/null
+++ b/.github/workflows/checkov.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C h e c k o v G i t H u b W o r k f l o w
+# ============================================================================ #
+
+# Static analysis of Terraform code - publishes report to GitHub Security tab
+
+# https://github.com/bridgecrewio/checkov-action
+
+---
+name: Checkov
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+jobs:
+ checkov:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Checkov
+ uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml
new file mode 100644
index 000000000..168d4914c
--- /dev/null
+++ b/.github/workflows/codeowners.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C o d e O w n e r s
+# ============================================================================ #
+
+---
+name: CodeOwners
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - CODEOWNERS
+ - .github/CODEOWNERS
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - CODEOWNERS
+ - .github/CODEOWNERS
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ validate:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Validate CODEOWNERS
+ uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 000000000..5e74ab863
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,63 @@
+---
+name: "CodeQL"
+
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ # The branches below must be a subset of the branches above
+ branches:
+ - master
+ schedule:
+ - cron: '37 15 * * 4'
+
+jobs:
+ analyze:
+ name: Analyze
+ runs-on: ubuntu-latest
+ permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+ strategy:
+ fail-fast: false
+ matrix:
+ language:
+ - python
+ # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+ # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+
+ # Initializes the CodeQL tools for scanning.
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@v1
+ with:
+ languages: ${{ matrix.language }}
+ # If you wish to specify custom queries, you can do so here or in a config file.
+ # By default, queries listed here will override any specified in a config file.
+ # Prefix the list here with "+" to use these queries and those in the config file.
+ # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+ # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
+ # If this step fails, then you should remove it and run the build manually (see below)
+ - name: Autobuild
+ uses: github/codeql-action/autobuild@v1
+
+ # Command-line programs to run using the OS shell.
+ # https://git.io/JvXDl
+
+ # If the Autobuild fails above, remove it and uncomment the following three lines
+ # and modify them (or add more) to build your code if your project
+ # uses a compiled language
+
+ #- run: |
+ # make bootstrap
+ # make release
+
+ - name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v1
diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml
new file mode 100644
index 000000000..d84cb16b1
--- /dev/null
+++ b/.github/workflows/debian.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:latest
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml
new file mode 100644
index 000000000..e857f280f
--- /dev/null
+++ b/.github/workflows/debian_10.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 10
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_10.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:10
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml
new file mode 100644
index 000000000..ad765f1ad
--- /dev/null
+++ b/.github/workflows/debian_11.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 11
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_11.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:11
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml
new file mode 100644
index 000000000..dfbc4ec97
--- /dev/null
+++ b/.github/workflows/debian_12.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 12
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_12.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:12
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled
new file mode 100644
index 000000000..f3062b6f8
--- /dev/null
+++ b/.github/workflows/debian_6.yaml.disabled
@@ -0,0 +1,50 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 6
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:6
+ # causes nodejs errors
+ #caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled
new file mode 100644
index 000000000..1b0e00295
--- /dev/null
+++ b/.github/workflows/debian_7.yaml.disabled
@@ -0,0 +1,50 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 7
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:7
+ # causes nodejs errors
+ #caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_8.yaml.disabled b/.github/workflows/debian_8.yaml.disabled
new file mode 100644
index 000000000..b41ead7d3
--- /dev/null
+++ b/.github/workflows/debian_8.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 8
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_8.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:8
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_9.yaml.disabled b/.github/workflows/debian_9.yaml.disabled
new file mode 100644
index 000000000..451348091
--- /dev/null
+++ b/.github/workflows/debian_9.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 9
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_9.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:9
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml
new file mode 100644
index 000000000..7119d6852
--- /dev/null
+++ b/.github/workflows/docker_pytools_alpine.yaml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Alpine)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:alpine
+ ghcr.io/harisekhon/pytools:alpine
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-alpine
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml
new file mode 100644
index 000000000..24f8f21dd
--- /dev/null
+++ b/.github/workflows/docker_pytools_debian.yaml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Debian)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:debian
+ ghcr.io/harisekhon/pytools:debian
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-debian
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml
new file mode 100644
index 000000000..ad1a5ed8d
--- /dev/null
+++ b/.github/workflows/docker_pytools_fedora.yaml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Fedora)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:fedora
+ ghcr.io/harisekhon/pytools:fedora
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-fedora
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml
new file mode 100644
index 000000000..884917919
--- /dev/null
+++ b/.github/workflows/docker_pytools_ubuntu.yaml
@@ -0,0 +1,126 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Ubuntu)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:latest
+ harisekhon/pytools:ubuntu
+ ghcr.io/harisekhon/pytools:latest
+ ghcr.io/harisekhon/pytools:ubuntu
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-ubuntu
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml
new file mode 100644
index 000000000..5a4f9a9b4
--- /dev/null
+++ b/.github/workflows/fedora.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Fedora
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/fedora.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: fedora
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml
new file mode 100644
index 000000000..6b228ca33
--- /dev/null
+++ b/.github/workflows/fork-sync.yaml
@@ -0,0 +1,48 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# F o r k S y n c
+# ============================================================================ #
+
+# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch
+
+---
+name: Fork Sync
+
+on: # yamllint disable-line rule:truthy
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 */3 * * *'
+
+permissions:
+ contents: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: false
+
+jobs:
+ fork_sync:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == true
+ if: github.repository_owner != 'HariSekhon'
+ name: Fork Sync
+ uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml
new file mode 100644
index 000000000..c74fa2a0b
--- /dev/null
+++ b/.github/workflows/fork-update-pr.yaml
@@ -0,0 +1,51 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# F o r k U p d a t e P R
+# ============================================================================ #
+
+# For a fork of the original repo, activate to keep its branches up to date via Pull Requests
+#
+# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date
+
+---
+name: Fork Update PR
+
+on: # yamllint disable-line rule:truthy
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 10 * * 1'
+
+permissions:
+ contents: write
+ pull-requests: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: false
+
+jobs:
+ fork_update_pr:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == true
+ if: github.repository_owner != 'HariSekhon'
+ name: Fork Update PR
+ uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ghcr_python_ubuntu.yaml.disabled b/.github/workflows/ghcr_python_ubuntu.yaml.disabled
new file mode 100644
index 000000000..c4503bc13
--- /dev/null
+++ b/.github/workflows/ghcr_python_ubuntu.yaml.disabled
@@ -0,0 +1,36 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-09 18:07:10 +0000 (Wed, 09 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: GHCR Build (Ubuntu)
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ workflow_dispatch:
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build_ghcr.yaml@master
+ with:
+ image: pytools
+ tags: ubuntu latest
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-ubuntu
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml
new file mode 100644
index 000000000..1b0fa15b4
--- /dev/null
+++ b/.github/workflows/grype.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G r y p e
+# ============================================================================ #
+
+---
+name: Grype
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ Grype:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Grype
+ uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml
new file mode 100644
index 000000000..83fa5ef83
--- /dev/null
+++ b/.github/workflows/json.yaml
@@ -0,0 +1,58 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# J S O N
+# ============================================================================ #
+
+# Validate any JSON files found in the repo
+
+---
+name: JSON
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.json'
+ - .github/workflows/json.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.json'
+ - .github/workflows/json.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+jobs:
+ check_json:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Check JSON
+ uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml
new file mode 100644
index 000000000..c389f5df0
--- /dev/null
+++ b/.github/workflows/kics.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# K i c s
+# ============================================================================ #
+
+---
+name: Kics
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ kics:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Kics
+ uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml
new file mode 100644
index 000000000..438dd184e
--- /dev/null
+++ b/.github/workflows/mac.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Mac
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ runs-on: macos-latest
+ caches: brew pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml
new file mode 100644
index 000000000..cc392b105
--- /dev/null
+++ b/.github/workflows/mac_11.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Mac 11
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ runs-on: macos-11
+ caches: brew pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml
new file mode 100644
index 000000000..0bcd6e6fd
--- /dev/null
+++ b/.github/workflows/mac_12.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Mac 12
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ runs-on: macos-12
+ caches: brew pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml
new file mode 100644
index 000000000..72e301ba0
--- /dev/null
+++ b/.github/workflows/markdown.yaml
@@ -0,0 +1,54 @@
+#
+# Author: Hari Sekhon
+# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# M a r k D o w n
+# ============================================================================ #
+
+---
+name: Markdown
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.md'
+ - .mdlrc
+ - .mdl.rb
+ - .markdownlint.rb
+ - .github/workflows/markdown.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.md'
+ - .mdlrc
+ - .mdl.rb
+ - .markdownlint.rb
+ - .github/workflows/markdown.yaml
+ workflow_dispatch:
+
+permissions:
+ contents: read
+ pull-requests: read
+
+jobs:
+ Markdown:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Markdown
+ uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master
diff --git a/.github/workflows/pypy2.yaml.disabled b/.github/workflows/pypy2.yaml.disabled
new file mode 100644
index 000000000..01080bcc8
--- /dev/null
+++ b/.github/workflows/pypy2.yaml.disabled
@@ -0,0 +1,56 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: PyPy 2
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: PyPy2
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: pypy2
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/pypy3.yaml.disabled b/.github/workflows/pypy3.yaml.disabled
new file mode 100644
index 000000000..101b36751
--- /dev/null
+++ b/.github/workflows/pypy3.yaml.disabled
@@ -0,0 +1,56 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: PyPy 3
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: PyPy3
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: pypy3
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python2.7.yaml.disabled b/.github/workflows/python2.7.yaml.disabled
new file mode 100644
index 000000000..1813931e7
--- /dev/null
+++ b/.github/workflows/python2.7.yaml.disabled
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 2.7
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python2.7.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python2.7.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 2.7
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 2.7
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml
new file mode 100644
index 000000000..8ec3008b9
--- /dev/null
+++ b/.github/workflows/python3.10.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.10
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.10.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.10.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.10
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: "3.10"
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml
new file mode 100644
index 000000000..0f27b0076
--- /dev/null
+++ b/.github/workflows/python3.11.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.11
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.11.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.11.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.11
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: "3.11"
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.6.yaml.disabled b/.github/workflows/python3.6.yaml.disabled
new file mode 100644
index 000000000..91bd3e673
--- /dev/null
+++ b/.github/workflows/python3.6.yaml.disabled
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.6
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.6.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.6.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.6
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.6
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml
new file mode 100644
index 000000000..73a0045c7
--- /dev/null
+++ b/.github/workflows/python3.7.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.7
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.7.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.7.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.7
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.7
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml
new file mode 100644
index 000000000..abcbc4c68
--- /dev/null
+++ b/.github/workflows/python3.8.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.8
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.8.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.8.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.8
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.8
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml
new file mode 100644
index 000000000..1ce7eb238
--- /dev/null
+++ b/.github/workflows/python3.9.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.9
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.9.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.9.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.9
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.9
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml
new file mode 100644
index 000000000..c2f614222
--- /dev/null
+++ b/.github/workflows/semgrep-cloud.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e m g r e p C l o u d W o r k f l o w
+# ============================================================================ #
+
+# Logs results to https://semgrep.dev/
+
+---
+name: Semgrep Cloud
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ semgrep:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Semgrep Cloud
+ uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master
+ secrets:
+ SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml
new file mode 100644
index 000000000..04a3cf197
--- /dev/null
+++ b/.github/workflows/semgrep.yaml
@@ -0,0 +1,64 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e m g r e p G i t H u b W o r k f l o w
+# ============================================================================ #
+
+# Generates code scanning alerts in GitHub's Security tab -> Code scanning alerts
+
+# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions
+
+---
+name: Semgrep
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ semgrep:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Semgrep GitHub Security Tab
+ uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml
new file mode 100644
index 000000000..246087bd4
--- /dev/null
+++ b/.github/workflows/shellcheck.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S h e l l C h e c k
+# ============================================================================ #
+
+# Validate any shell scripts found in the repo
+
+---
+name: ShellCheck
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.sh'
+ - .github/workflows/shellcheck.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.sh'
+ - .github/workflows/shellcheck.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ shellcheck:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: ShellCheck
+ uses: HariSekhon/GitHub-Actions/.github/workflows/shellcheck.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml
new file mode 100644
index 000000000..95b2571ca
--- /dev/null
+++ b/.github/workflows/trivy.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# T r i v y
+# ============================================================================ #
+
+# Scan files in the local repo
+
+---
+name: Trivy
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ trivy:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Trivy
+ uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml
new file mode 100644
index 000000000..fe863f875
--- /dev/null
+++ b/.github/workflows/ubuntu.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:latest
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_14.04.yaml.disabled b/.github/workflows/ubuntu_14.04.yaml.disabled
new file mode 100644
index 000000000..1442d675d
--- /dev/null
+++ b/.github/workflows/ubuntu_14.04.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 14.04
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_14.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:14.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_16.04.yaml.disabled b/.github/workflows/ubuntu_16.04.yaml.disabled
new file mode 100644
index 000000000..a9655835d
--- /dev/null
+++ b/.github/workflows/ubuntu_16.04.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 16.04
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_16.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:16.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml.disabled
new file mode 100644
index 000000000..4b510ba1e
--- /dev/null
+++ b/.github/workflows/ubuntu_18.04.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 18.04
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_18.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:18.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml
new file mode 100644
index 000000000..47d0512ac
--- /dev/null
+++ b/.github/workflows/ubuntu_20.04.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 20.04
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_20.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:20.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml
new file mode 100644
index 000000000..ba8a549b1
--- /dev/null
+++ b/.github/workflows/ubuntu_22.04.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 22.04
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_22.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:22.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml
new file mode 100644
index 000000000..0037e4ba3
--- /dev/null
+++ b/.github/workflows/ubuntu_github.yaml
@@ -0,0 +1,84 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: GitHub Actions Ubuntu
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_github.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
new file mode 100644
index 000000000..ab6ab9638
--- /dev/null
+++ b/.github/workflows/validate.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# V a l i d a t i o n
+# ============================================================================ #
+
+# Run all custom validations against files in the repo
+
+---
+name: Validation
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ #schedule:
+ # - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ validate:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Validate
+ uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml
new file mode 100644
index 000000000..3f6265a6a
--- /dev/null
+++ b/.github/workflows/xml.yaml
@@ -0,0 +1,56 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# X M L
+# ============================================================================ #
+
+# Validate any XML files found in the repo
+
+---
+name: XML
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.xml'
+ - .github/workflows/xml.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.xml'
+ - .github/workflows/xml.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+jobs:
+ check_xml:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Check XML
+ uses: HariSekhon/GitHub-Actions/.github/workflows/xml.yaml@master
diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml
new file mode 100644
index 000000000..f2a4ca129
--- /dev/null
+++ b/.github/workflows/yaml.yaml
@@ -0,0 +1,64 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# Y A M L
+# ============================================================================ #
+
+# Validate any YAML files found in the repo
+
+---
+name: YAML
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.yml'
+ - '**/*.yaml'
+ - .github/workflows/yaml.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.yml'
+ - '**/*.yaml'
+ - .github/workflows/yaml.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ check_yaml:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Check YAML
+ uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.gitignore b/.gitignore
index 525ea88b7..3722dc711 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,10 +1,14 @@
+# vim: filetype=conf
+
# This doesn't actually work on already committed files, git update-index --assume-unchanged is set in Makefile instead
anonymize_custom.conf
+
solr-env.sh
spark-deps.zip
*.pyc
*.pyo
*.jyc
+*.jyo
*.avro
*.parquet
*.log
@@ -22,3 +26,135 @@ parquet-tools-*-bin.zip
tests/docker/registry.crt
tests/docker/registry.csr
tests/docker/registry.key
+
+# ============================================================================ #
+# Created by https://www.gitignore.io/api/python
+# Edit at https://www.gitignore.io/?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+#lib/ # breaks local lib/ change tracking
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# End of https://www.gitignore.io/api/python
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 000000000..625220b32
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,30 @@
+# vim:ts=2:sts=2:sw=2:et
+#
+# Author: Hari Sekhon
+# Date: Sun Feb 23 19:02:10 2020 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G i t L a b C I
+# ============================================================================ #
+
+# https://docs.gitlab.com/ee/ci/yaml/README.html
+
+#include: '.gitlab/*.y*ml'
+
+image: ubuntu:18.04
+
+job:
+ before_script:
+ - setup/ci_bootstrap.sh
+ script:
+ - make init && make ci test
diff --git a/.gitmodules b/.gitmodules
index 37537e726..30fa60a67 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,8 +1,16 @@
[submodule "pylib"]
path = pylib
- url = https://github.com/harisekhon/pylib
+ url = https://github.com/HariSekhon/pylib
branch = master
[submodule "bash-tools"]
path = bash-tools
- url = https://github.com/harisekhon/bash-tools
+ url = https://github.com/HariSekhon/DevOps-Bash-tools
+ branch = master
+[submodule "sql"]
+ path = sql
+ url = https://github.com/HariSekhon/SQL-scripts
+ branch = master
+[submodule "templates"]
+ path = templates
+ url = https://github.com/HariSekhon/Templates
branch = master
diff --git a/.ipython-notebook-pyspark.00-pyspark-setup.py b/.ipython-notebook-pyspark.00-pyspark-setup.py
index 4dde897da..1de53b3fb 100644
--- a/.ipython-notebook-pyspark.00-pyspark-setup.py
+++ b/.ipython-notebook-pyspark.00-pyspark-setup.py
@@ -24,4 +24,4 @@
sys.path.insert(0, os.path.join(spark_home, 'python'))
for lib in glob.glob(os.path.join(spark_home, 'python/lib/py4j-*-src.zip')):
sys.path.insert(0, lib)
-execfile(os.path.join(spark_home, 'python/pyspark/shell.py'))
+execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) # pylint: disable=undefined-variable
diff --git a/.mdl.rb b/.mdl.rb
new file mode 100644
index 000000000..f8f9b004c
--- /dev/null
+++ b/.mdl.rb
@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+# vim:ts=4:sts=4:sw=4:et:filetype=ruby
+#
+# Author: Hari Sekhon
+# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024)
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+all
+#exclude_rule 'MD001'
+#exclude_rule 'MD003'
+#exclude_rule 'MD005'
+exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af
+#exclude_rule 'MD012'
+exclude_rule 'MD013' # long lines cannot be split if they are URLs
+#exclude_rule 'MD022'
+#exclude_rule 'MD025'
+#exclude_rule 'MD031'
+#exclude_rule 'MD032'
+exclude_rule 'MD033' # inline HTML is important for formatting
+exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines
+#exclude_rule 'MD039'
+#exclude_rule 'MD056'
diff --git a/.mdlrc b/.mdlrc
new file mode 100644
index 000000000..27e5b6895
--- /dev/null
+++ b/.mdlrc
@@ -0,0 +1,5 @@
+mdlrc_dir = File.expand_path('..', __FILE__)
+
+style_file = File.join(mdlrc_dir, '.mdl.rb')
+
+style style_file
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000..787fb79ea
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,74 @@
+#
+# Author: Hari Sekhon
+# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# P r e - C o m m i t
+# ============================================================================ #
+
+---
+fail_fast: false
+#exclude: *.tmp$
+
+repos:
+
+ # will accept anything that 'git clone' understands
+ # this means you can set this to a local git repo to develop your own hook repos interactively
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.6.0
+ hooks:
+ - id: check-yaml
+ # Common errors
+ #- id: end-of-file-fixer # ruins .gitignore Icon\r
+ - id: trailing-whitespace
+ args: [--markdown-linebreak-ext=md]
+ # Git style
+ - id: check-added-large-files
+ - id: check-merge-conflict
+ - id: check-vcs-permalinks
+ #- id: forbid-new-submodules
+ # Cross platform
+ - id: check-case-conflict
+ - id: mixed-line-ending
+ args: [--fix=lf]
+ # Security
+ - id: detect-aws-credentials
+ args: ['--allow-missing-credentials']
+
+ # rewrites python files with useless changes like changing single quotes to double quotes
+ #- repo: https://github.com/psf/black
+ # rev: 24.8.0
+ # hooks:
+ # - id: black
+
+ # Git secrets Leaks
+ - repo: https://github.com/awslabs/git-secrets.git
+ # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error:
+ #
+ # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file
+ #
+ rev: 5357e18
+ hooks:
+ - id: git-secrets
+
+ - repo: https://github.com/markdownlint/markdownlint
+ rev: v0.12.0
+ hooks:
+ - id: markdownlint
+ name: Markdownlint
+ description: Run markdownlint on your Markdown files
+ entry: mdl
+ args: [-s, .mdl.rb]
+ language: ruby
+ files: \.(md|mdown|markdown)$
diff --git a/.pylintrc b/.pylintrc
index 65e97ab22..2006619da 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -1,378 +1,655 @@
-[MASTER]
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# P y L i n t C o n f i g
+# ============================================================================ #
+
+# pylint --generate-rcfile >> .pylintrc
+
+[MAIN]
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Clear in-memory caches upon conclusion of linting. Useful if running pylint
+# in a server-like mode.
+clear-cache-post-run=no
+
+# Load and enable all available extensions. Use --list-extensions to see a list
+# all available extensions.
+#enable-all-extensions=
+
+# In error mode, messages with a category besides ERROR or FATAL are
+# suppressed, and no reports are done by default. Error mode is compatible with
+# disabling specific errors.
+#errors-only=
+
+# Always return a 0 (non-error) status code, even if lint errors are found.
+# This is primarily useful in continuous integration scripts.
+#exit-zero=
-# Specify a configuration file.
-#rcfile=
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-allow-list=
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
+# for backward compatibility.)
+extension-pkg-whitelist=
+
+# Return non-zero exit code if any of these messages/categories are detected,
+# even if score is above --fail-under value. Syntax same as enable. Messages
+# specified are enabled, while categories only check already-enabled messages.
+fail-on=
+
+# Specify a score threshold under which the program will exit with error.
+fail-under=10
+
+# Interpret the stdin as a python script, whose filename needs to be passed as
+# the module_or_package argument.
+#from-stdin=
+
+# Files or directories to be skipped. They should be base names, not paths.
+ignore=CVS
+
+# Add files or directories matching the regular expressions patterns to the
+# ignore-list. The regex matches against paths and can be in Posix or Windows
+# format. Because '\\' represents the directory delimiter on Windows systems,
+# it can't be used as an escape character.
+ignore-paths=
+
+# Files or directories matching the regular expression patterns are skipped.
+# The regex matches against base names, not paths. The default value ignores
+# Emacs file locks
+ignore-patterns=^\.#
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis). It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=CVS
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use, and will cap the count on Windows to
+# avoid hangs.
+jobs=1
-# Pickle collected data for later comparisons.
-persistent=yes
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
-# List of plugins (as comma separated values of python modules names) to load,
+# List of plugins (as comma separated values of python module names) to load,
# usually to register additional checkers.
load-plugins=
-# Use multiple processes to speed up Pylint.
-jobs=1
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Minimum Python version to use for version dependent checks. Will default to
+# the version used to run pylint.
+py-version=3.11
+
+# Discover python modules and packages in the file system subtree.
+recursive=no
+
+# Add paths to the list of the source roots. Supports globbing patterns. The
+# source root is an absolute path or a path relative to the current working
+# directory used to determine a package namespace for modules located under the
+# source root.
+source-roots=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
-# A comma-separated list of package or module names from where C extensions may
-# be loaded. Extensions are loading into the active Python interpreter and may
-# run arbitrary code
-extension-pkg-whitelist=
+# In verbose mode, extra non-checker-related info will be displayed.
+#verbose=
-# Allow optimization of some AST trees. This will activate a peephole AST
-# optimizer, which will apply various small optimizations. For instance, it can
-# be used to obtain the result of joining multiple strings with the addition
-# operator. Joining a lot of strings can lead to a maximum recursion error in
-# Pylint and this flag can prevent that. It has one side effect, the resulting
-# AST will be different than the one from reality.
-optimize-ast=no
+[BASIC]
-[MESSAGES CONTROL]
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
-# Only show warnings with the listed confidence levels. Leave empty to show
-# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
-confidence=
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style. If left empty, argument names will be checked with the set
+# naming style.
+#argument-rgx=
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time. See also the "--disable" option for examples.
-#enable=
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once).You can also use "--disable=all" to
-# disable everything first and then reenable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use"--disable=all --enable=classes
-# --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style. If left empty, attribute names will be checked with the set naming
+# style.
+#attr-rgx=
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+ bar,
+ baz,
+ toto,
+ tutu,
+ tata
-[REPORTS]
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
-# Set the output format. Available formats are text, parseable, colorized, msvs
-# (visual studio) and html. You can also give a reporter class, eg
-# mypackage.mymodule.MyReporterClass.
-output-format=text
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style. If left empty, class attribute names will be checked
+# with the set naming style.
+#class-attribute-rgx=
-# Tells whether to display a full report or only the messages
-reports=yes
+# Naming style matching correct class constant names.
+class-const-naming-style=UPPER_CASE
-# Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables errors warning, statement which
-# respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+# Regular expression matching correct class constant names. Overrides class-
+# const-naming-style. If left empty, class constant names will be checked with
+# the set naming style.
+#class-const-rgx=
-# Template used to display messages. This is a python new-style format string
-# used to format the message information. See doc for all details
-#msg-template=
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+# Regular expression matching correct class names. Overrides class-naming-
+# style. If left empty, class names will be checked with the set naming style.
+#class-rgx=
-[BASIC]
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style. If left empty, constant names will be checked with the set naming
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style. If left empty, function names will be checked with the set
+# naming style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+ j,
+ k,
+ ex,
+ Run,
+ _
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style. If left empty, inline iteration names will be checked
+# with the set naming style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,input
+# Regular expression matching correct method names. Overrides method-naming-
+# style. If left empty, method names will be checked with the set naming style.
+#method-rgx=
-# Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_
+# Naming style matching correct module names.
+module-naming-style=snake_case
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,bar,baz,toto,tutu,tata
+# Regular expression matching correct module names. Overrides module-naming-
+# style. If left empty, module names will be checked with the set naming style.
+#module-rgx=
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
-# Include a hint for the correct naming format with invalid-name
-include-naming-hint=no
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
-# Regular expression matching correct function names
-function-rgx=[a-z_][a-z0-9_]{2,30}$
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
-# Naming hint for function names
-function-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Regular expression matching correct type alias names. If left empty, type
+# alias names will be checked with the set naming style.
+#typealias-rgx=
-# Regular expression matching correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
+# Regular expression matching correct type variable names. If left empty, type
+# variable names will be checked with the set naming style.
+#typevar-rgx=
-# Naming hint for variable names
-variable-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
-# Regular expression matching correct constant names
-const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style. If left empty, variable names will be checked with the set
+# naming style.
+#variable-rgx=
-# Naming hint for constant names
-const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
-# Regular expression matching correct attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,30}$
+[CLASSES]
-# Naming hint for attribute names
-attr-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Warn about protected attribute access inside special methods
+check-protected-access-in-special-methods=no
-# Regular expression matching correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+ __new__,
+ setUp,
+ asyncSetUp,
+ __post_init__
-# Naming hint for argument names
-argument-name-hint=[a-z_][a-z0-9_]{2,30}$
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit
-# Regular expression matching correct class attribute names
-class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
-# Naming hint for class attribute names
-class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
-# Regular expression matching correct inline iteration names
-inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
-# Naming hint for inline iteration names
-inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+[DESIGN]
-# Regular expression matching correct class names
-class-rgx=[A-Z_][a-zA-Z0-9]+$
+# List of regular expressions of class ancestor names to ignore when counting
+# public methods (see R0903)
+exclude-too-few-public-methods=
-# Naming hint for class names
-class-name-hint=[A-Z_][a-zA-Z0-9]+$
+# List of qualified class names to ignore when counting class parents (see
+# R0901)
+ignored-parents=
-# Regular expression matching correct module names
-module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+# Maximum number of arguments for function / method.
+max-args=5
-# Naming hint for module names
-module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
-# Regular expression matching correct method names
-method-rgx=[a-z_][a-z0-9_]{2,30}$
+# Maximum number of boolean expressions in an if statement (see R0916).
+max-bool-expr=5
-# Naming hint for method names
-method-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Maximum number of branch for function / method body.
+max-branches=12
-# Regular expression which should only match function or class names that do
-# not require a docstring.
-no-docstring-rgx=^_
+# Maximum number of locals for function / method body.
+max-locals=15
-# Minimum line length for functions/classes that require docstrings, shorter
-# ones are exempt.
-docstring-min-length=-1
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
-[ELIF]
+# Maximum number of return / yield for function / method body.
+max-returns=6
-# Maximum number of nested blocks for function / method body
-max-nested-blocks=5
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when caught.
+overgeneral-exceptions=builtins.BaseException,builtins.Exception
[FORMAT]
-# Maximum number of characters on a single line.
-max-line-length=120
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )??$
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string=' '
+
+# Maximum number of characters on a single line.
+max-line-length=120
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,dict-separator
-# Maximum number of lines in a module
-max-module-lines=1000
+[IMPORTS]
-# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
-# tab).
-indent-string=' '
+# List of modules that can be imported at any level, not just the top level
+# one.
+allow-any-import-level=
-# Number of spaces of indent required inside a hanging or continued line.
-indent-after-paren=4
+# Allow explicit reexports by alias from a package __init__.
+allow-reexport-from-package=no
-# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
-expected-line-ending-format=
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=
+
+# Output a graph (.gv or any supported image format) of external dependencies
+# to the given file (report RP0402 must not be disabled).
+ext-import-graph=
+
+# Output a graph (.gv or any supported image format) of all (i.e. internal and
+# external) dependencies to the given file (report RP0402 must not be
+# disabled).
+import-graph=
+
+# Output a graph (.gv or any supported image format) of internal dependencies
+# to the given file (report RP0402 must not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Couples of modules and preferred modules, separated by a comma.
+preferred-modules=
[LOGGING]
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
# Logging modules to check that the string format arguments are in logging
-# function parameter format
+# function parameter format.
logging-modules=logging
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE,
+# UNDEFINED.
+confidence=HIGH,
+ CONTROL_FLOW,
+ INFERENCE,
+ INFERENCE_FAILURE,
+ UNDEFINED
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then re-enable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=raw-checker-failed,
+ bad-inline-option,
+ locally-disabled,
+ file-ignored,
+ suppressed-message,
+ useless-suppression,
+ deprecated-pragma,
+ use-symbolic-message-instead,
+ missing-class-docstring,
+ missing-function-docstring,
+ super-with-arguments,
+ consider-using-f-string
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[METHOD_ARGS]
+
+# List of qualified names (i.e., library.method) which require a timeout
+# parameter e.g. 'requests.api.get,requests.api.post'
+timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request
+
+
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
+notes=FIXME,
+ XXX,
+ TODO
+# Regular expression of note tags to take in consideration.
+notes-rgx=
-[SIMILARITIES]
-# Minimum lines number of a similarity.
-min-similarity-lines=4
+[REFACTORING]
-# Ignore comments when computing similarities.
-ignore-comments=yes
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit,argparse.parse_error
-# Ignore imports when computing similarities.
-ignore-imports=no
+[REPORTS]
-[SPELLING]
+# Python expression which should return a score less than or equal to 10. You
+# have access to the variables 'fatal', 'error', 'warning', 'refactor',
+# 'convention', and 'info' which contain the number of messages in each
+# category, as well as 'statement' which is the total number of statements
+# analyzed. This score is used by the global evaluation report (RP0004).
+evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))
-# Spelling dictionary name. Available dictionaries: none. To make it working
-# install python-enchant package.
-spelling-dict=
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+msg-template=
-# List of comma separated words that should not be checked.
-spelling-ignore-words=
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+#output-format=
-# A path to a file that contains private dictionary; one word per line.
-spelling-private-dict-file=
+# Tells whether to display a full report or only the messages.
+reports=no
-# Tells whether to store unknown words to indicated private dictionary in
-# --spelling-private-dict-file option instead of raising a message.
-spelling-store-unknown-words=no
+# Activate the evaluation score.
+score=yes
-[TYPECHECK]
+[SIMILARITIES]
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
+# Comments are removed from the similarity computation
+ignore-comments=yes
-# List of module names for which member attributes should not be checked
-# (useful for modules/projects where namespaces are manipulated during runtime
-# and thus existing member attributes cannot be deduced by static analysis. It
-# supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+# Docstrings are removed from the similarity computation
+ignore-docstrings=yes
-# List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamically set). This supports can work
-# with qualified names.
-ignored-classes=
+# Imports are removed from the similarity computation
+ignore-imports=yes
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-generated-members=
+# Signatures are removed from the similarity computation
+ignore-signatures=yes
+# Minimum lines number of a similarity.
+min-similarity-lines=4
-[VARIABLES]
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
+[SPELLING]
-# A regular expression matching the name of dummy variables (i.e. expectedly
-# not used).
-dummy-variables-rgx=_$|dummy
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid to define new builtins when possible.
-additional-builtins=
+# Spelling dictionary name. No available dictionaries : You need to install
+# both the python package and the system dependency for enchant to work..
+spelling-dict=
-# List of strings which can identify a callback function by name. A callback
-# name must start or end with one of those strings.
-callbacks=cb_,_cb
+# List of comma separated words that should be considered directives if they
+# appear at the beginning of a comment and should not be checked.
+spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
-[CLASSES]
+# A path to a file that contains the private dictionary; one word per line.
+spelling-private-dict-file=
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,__new__,setUp
+# Tells whether to store unknown words to the private dictionary (see the
+# --spelling-private-dict-file option) instead of raising a message.
+spelling-store-unknown-words=no
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
-# List of valid names for the first argument in a metaclass class method.
-valid-metaclass-classmethod-first-arg=mcs
+[STRING]
-# List of member names, which should be excluded from the protected access
-# warning.
-exclude-protected=_asdict,_fields,_replace,_source,_make
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
-[DESIGN]
-# Maximum number of arguments for function / method
-max-args=5
+[TYPECHECK]
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore
-ignored-argument-names=_.*
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
-# Maximum number of locals for function / method body
-max-locals=15
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
-# Maximum number of return / yield for function / method body
-max-returns=6
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
-# Maximum number of branch for function / method body
-max-branches=12
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
-# Maximum number of statements in function / method body
-max-statements=50
+# List of symbolic message names to ignore for Mixin members.
+ignored-checks-for-mixins=no-member,
+ not-async-context-manager,
+ not-context-manager,
+ attribute-defined-outside-init
-# Maximum number of parents for a class (see R0901).
-max-parents=7
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace
-# Maximum number of attributes for a class (see R0902).
-max-attributes=7
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=20
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
-# Maximum number of boolean expressions in a if statement
-max-bool-expr=5
+# Regex pattern to define which classes are considered mixins.
+mixin-class-rgx=.*[Mm]ixin
+# List of decorators that change the signature of a decorated function.
+signature-mutators=
-[IMPORTS]
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,TERMIOS,Bastion,rexec
+[VARIABLES]
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled)
-import-graph=
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled)
-ext-import-graph=
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled)
-int-import-graph=
+# List of names allowed to shadow builtins
+allowed-redefined-builtins=
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+ _cb
-[EXCEPTIONS]
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
diff --git a/.pyup.yml b/.pyup.yml
new file mode 100644
index 000000000..aabbdd2ef
--- /dev/null
+++ b/.pyup.yml
@@ -0,0 +1,4 @@
+# https://pyup.io/docs/configuration/
+
+update: insecure
+assignees: HariSekhon
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
new file mode 100644
index 000000000..59aa80836
--- /dev/null
+++ b/.semaphore/semaphore.yml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e m a p h o r e C I
+# ============================================================================ #
+
+# https://docs.semaphoreci.com/reference/pipeline-yaml-reference/
+
+version: v1.0
+name: DevOps-Python-tools
+agent:
+ # https://docs.semaphoreci.com/reference/machine-types#linux
+ machine:
+ type: e1-standard-2
+ os_image: ubuntu2004
+execution_time_limit:
+ hours: 3
+blocks:
+ - name: Linux build
+ run:
+ when: "branch = 'master' AND change_in('/', {exclude: ['**/*.md']})"
+ #execution_time_limit:
+ # hours: 2
+ task:
+ #env_vars:
+ # $PATH selects /usr/bin/python and /usr/local/bin/pip which are mismatched versions of Python
+ #- name: PYTHON
+ # value: python3
+ #- name: PIP
+ # value: pip3
+ prologue:
+ commands:
+ - cache restore
+ # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server
+ #
+ # causes error:
+ #
+ # Not replacing deleted config file /etc/ssh/sshd_config
+ #
+ #- sudo rm -f /etc/ssh/sshd_config
+ - export DEBIAN_FRONTEND=noninteractive
+ - sudo -E apt-get update
+ - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew"
+ - sudo dpkg --configure -a --force-confmiss --force-confnew
+ #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections
+ #- sudo dpkg --configure -a --force-confdef --force-confold
+ #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"
+ - sudo apt-get install -y openssh-server
+ # each job is separate and could be run on a separate machine so all steps must be together
+ jobs:
+ - name: build
+ commands:
+ - checkout
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
+ epilogue:
+ commands:
+ - cache store
+ - name: Mac build
+ run:
+ when: "branch = 'master'"
+ task:
+ # because otherwise on Mac it uses /usr/bin/python (2.7) but /usr/local/bin/pip (python 3.8)
+ #env_vars:
+ # to match /usr/local/bin/pip version from $PATH
+ #- name: PYTHON
+ # value: python3
+ # must be quoted to force string, otherwise pipeline fails to run with this parsing error:
+ # Error: [{"Type mismatch. Expected String but got Integer.", "#/blocks/1/task/env_vars/1/value"}]
+ #- name: DEBUG
+ # value: "1"
+ agent:
+ # https://docs.semaphoreci.com/reference/machine-types#macos
+ machine:
+ type: a1-standard-4
+ os_image: macos-xcode15
+ prologue:
+ commands:
+ - cache restore
+ # fix for:
+ # pip is configured with locations that require TLS/SSL, however the ssl module in Python is not available.
+ - brew install openssl
+ - brew reinstall python
+ - brew reinstall wget
+ # avoid Mac SSL errors:
+ #
+ # ERROR: Loading command: install (LoadError)
+ # dlopen(/Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle, 9): Library not loaded: /usr/local/opt/openssl/lib/libssl.1.0.0.dylib
+ # Referenced from: /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle
+ # Reason: image not found - /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle
+ # ERROR: While executing gem ... (NoMethodError)
+ # undefined method `invoke_with_build_args' for nil:NilClass#
+ #
+ - rbenv global system
+ # also considered this:
+ # - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done
+ #
+ # fix for python vs pip version mismatch
+ - ln -svf -- /usr/local/bin/python3 /usr/local/bin/python
+ jobs:
+ - name: build
+ commands:
+ - checkout
+ - make init
+ - make ci
+ - make test
+ epilogue:
+ commands:
+ - cache store
diff --git a/.sonarcloud.properties b/.sonarcloud.properties
new file mode 100644
index 000000000..a4496869e
--- /dev/null
+++ b/.sonarcloud.properties
@@ -0,0 +1 @@
+sonar.host.url=https://sonarcloud.io
diff --git a/.travis.yml b/.travis.yml
index e64530f13..47f2f80b5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,46 +3,142 @@
# Author: Hari Sekhon
# Date: 2014-11-29 01:02:47 +0000 (Sat, 29 Nov 2014)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
-language: python
+# ============================================================================ #
+# T r a v i s C I
+# ============================================================================ #
+
+# https://docs.travis-ci.com/user/customizing-the-build/
+
+---
+version: ~> 1.0
+
+# ============================
+# Mac Python 2.7 not available - see https://travis-ci.org/HariSekhon/pylib/jobs/584226228
+#os:
+# - linux
+# - osx
+
+# =================================================
+# https://docs.travis-ci.com/user/languages/python/
python:
-# - "2.6"
- "2.7"
-# MySQL in lib doesn't build from pip in Python 3
-# - "3.2"
-# - "3.3"
-# - "3.4"
-# - "3.5"
-# python-krbV fails to compile on PyPy
-# - "pypy"
-# - "pypy3"
+ #- "3.4" supported by pip as of March 2019
+ - "3.5"
+ - "3.6"
+ - "3.7"
+ - "3.8"
+ - "pypy" # currently Python 2.7.13, PyPy 7.1.1
+ - "pypy3" # currently Python 3.6.1, PyPy 7.1.1-beta0
+
+# =============================================
+# https://docs.travis-ci.com/user/build-matrix/
+# https://docs.travis-ci.com/user/build-stages/matrix-expansion/
+matrix:
+ fast_finish: true
+ include:
+ # numpy has gone 2.7+ only now, so had to drop Python 2.6 support
+ - os: linux
+ language: python
+ python: "2.7"
+
+ - os: osx
+ language: generic # workaround since Mac doesn't have Python support yet, so install to system Python
+ # https://docs.travis-ci.com/user/reference/osx/
+ # macOS 10.15.7 - otherwise defaults to Mac macOS 10.13 with xcode9.4 otherwise - and HomeBrew update takes 50 minutes until the build times out :-/
+ osx_image: xcode12.2
+
+ - os: linux
+ language: python
+ python: "3.5"
+
+ - os: linux
+ language: python
+ python: "3.6"
+
+ - os: linux
+ language: python
+ python: "3.7"
+
+ - os: linux
+ language: python
+ python: "3.8"
-sudo: required
+ # python-krbV fails to compile on PyPy
+ #
+ # psutil doesn't build:
+ #
+ # RuntimeError: broken / incompatible Python implementation, see: https://github.com/giampaolo/psutil/issues/1659
+ #
+ #- os: linux
+ # language: python
+ # python: "pypy"
+ - os: linux
+ language: python
+ python: "pypy3"
+
+ # ===================================================================================================
+ # https://docs.travis-ci.com/user/multi-os/#allowing-failures-on-jobs-running-on-one-operating-system
+ allow_failures:
+ - python: "pypy"
+ - python: "pypy3"
+
+# =======================================
+# https://docs.travis-ci.com/user/docker/
+services:
+ - docker
+
+# ======================================================
+# https://docs.travis-ci.com/user/environment-variables/
env:
# - DEBUG=1
- DOCKER_COMPOSE_VERSION=1.16.1 PYTHONUNBUFFERED=1
+# ==============================================
+# https://docs.travis-ci.com/user/notifications/
notifications:
email: false
-branches:
- only:
- - master
+# =================================================================================
+# https://docs.travis-ci.com/user/customizing-the-build/#building-specific-branches
+# https://docs.travis-ci.com/user/conditional-builds-stages-jobs
+#branches:
+# only:
+# - master
-cache: pip
+# ========================================
+# https://docs.travis-ci.com/user/caching/
-services:
- - docker
+before_cache:
+ #- rm -f $HOME/.cache/pip/log/debug.log
+ # XXX: cache breaks pypy builds, so clear it
+ - rm -f $HOME/.cache/pip
+
+cache:
+ - pip
+ - directories:
+ - $HOME/.cache
+ - $HOME/.cpan
+ - $HOME/.cpanm
+ - $HOME/.gem
+
+# ==============================================
+# https://docs.travis-ci.com/user/job-lifecycle/
+
+# avoid package checksum mismatches when installing packages
+before_install:
+ - sudo rm -f "${TRAVIS_ROOT}/etc/apt/apt.conf.d/99-travis-apt-proxy"
+ - sudo rm -rf "${TRAVIS_ROOT}/var/lib/apt/lists/"*
install:
- travis_retry make
@@ -54,8 +150,3 @@ script:
after_success:
- coveralls
- - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/pytools/trigger/0e69c39f-ea1b-43c7-a97d-cef1252f1400/
- - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/alpine-github/trigger/8d9cb826-48df-439c-8c20-1975713064fc/
- - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/centos-github/trigger/316d1158-7ffb-49a4-a7bd-8e5456ba2d15/
- - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/debian-github/trigger/0cfb3fe7-2028-494b-a43b-068435e6a2b3/
- - curl --data {build:true} -X POST https://registry.hub.docker.com/u/harisekhon/ubuntu-github/trigger/abd4dbf0-14bc-454f-9cde-081ec014bc48/
diff --git a/.yamllint b/.yamllint
new file mode 100644
index 000000000..e29153b7f
--- /dev/null
+++ b/.yamllint
@@ -0,0 +1,33 @@
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+
+extends: default
+
+rules:
+ # error too many spaces after colon
+ colons: disable
+
+ # warning missing starting space in comment
+ comments: disable
+
+ # warning comment not indented like content
+ comments-indentation: disable
+
+ # warning missing document start "---"
+ document-start: disable
+
+ # error line too long (94 > 80 characters)
+ line-length: disable
diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 000000000..d582bd112
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,88 @@
+// vim:ts=4:sts=4:sw=4:et:filetype=groovy:syntax=groovy
+//
+// Author: Hari Sekhon
+// Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017)
+//
+// https://github.com/HariSekhon/DevOps-Python-tools
+//
+// License: see accompanying Hari Sekhon LICENSE file
+//
+// If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+//
+// https://www.linkedin.com/in/HariSekhon
+//
+
+// ========================================================================== //
+// J e n k i n s P i p e l i n e
+// ========================================================================== //
+
+// Epic Jenkinsfile template:
+//
+// https://github.com/HariSekhon/Templates/blob/master/Jenkinsfile
+
+
+// Official Documentation:
+//
+// https://jenkins.io/doc/book/pipeline/syntax/
+//
+// https://www.jenkins.io/doc/pipeline/steps/
+//
+// https://www.jenkins.io/doc/pipeline/steps/workflow-basic-steps/
+
+
+pipeline {
+ // to run on Docker or Kubernetes, see the master Jenkinsfile template listed at the top
+ agent any
+
+ options {
+ timestamps()
+
+ timeout(time: 2, unit: 'HOURS')
+ }
+
+ triggers {
+ cron('H 10 * * 1-5')
+ pollSCM('H/2 * * * *')
+ }
+
+ stages {
+ stage ('Checkout') {
+ steps {
+ checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/HariSekhon/DevOps-Python-tools']]])
+ }
+ }
+
+ stage('Build') {
+ steps {
+ echo "Running ${env.JOB_NAME} Build ${env.BUILD_ID} on ${env.JENKINS_URL}"
+ echo 'Building...'
+ timeout(time: 10, unit: 'MINUTES') {
+ retry(3) {
+// sh 'apt update -q'
+// sh 'apt install -qy make'
+// sh 'make init'
+ sh """
+ setup/ci_bootstrap.sh &&
+ make init
+ """
+ }
+ }
+ timeout(time: 180, unit: 'MINUTES') {
+ sh 'make ci'
+ }
+ }
+ }
+
+ stage('Test') {
+ options {
+ retry(2)
+ }
+ steps {
+ echo 'Testing...'
+ timeout(time: 120, unit: 'MINUTES') {
+ sh 'make test'
+ }
+ }
+ }
+ }
+}
diff --git a/LICENSE b/LICENSE
index 03b527051..4860c1cf5 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,45 +1,7 @@
-=======================================
-HARI SEKHON LICENSE Revision 2013112300
-=======================================
+Copyright 2015 Hari Sekhon
-Copyright (c) 2006 onwards, Hari Sekhon
-All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-Redistribution and use in source and binary forms, with or without
-modification, is permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-3. All advertising materials mentioning features or use of this software
- must display the following acknowledgement:
- This product includes software developed by Hari Sekhon.
-4. Neither the name Hari Sekhon nor any affiliates may be used to endorse or
- promote products derived from this software without specific prior written
- permission.
-5. Modifications may be released to the public only with prior written permission
- from Hari Sekhon. Forking on GitHub is permitted for the purpose of creating
- patch pull requests back to the original repository.
-6. Private modifications may be made to suit requirements, but any modifications
- to this work, whether publicly disclosed or not, must be sent back to
- Hari Sekhon via GitHub (https://github.com/harisekhon/devops-python-tools) or
- LinkedIn (https://www.linkedin.com/in/harisekhon)
- and must come under this same license. Any such modifications may be
- reincorporated for the improvement of this software.
-7. This work may not be sold without prior written permission from Hari Sekhon
-8. This license may change at any time and the latest revision supersedes
- all prior revisions.
-9. Alternative licensing must be agreed in writing with Hari Sekhon prior
- to public availability.
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-THIS SOFTWARE IS PROVIDED BY Hari Sekhon ''AS IS'' AND ANY
-EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL Hari Sekhon OR ANY AFFILIATED BODY BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/Makefile b/Makefile
old mode 100755
new mode 100644
index 6aa8b04c3..1d98ac524
--- a/Makefile
+++ b/Makefile
@@ -2,170 +2,181 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules to non-system python otherwise they're not found by programs.
# Better than modifying $PATH to put /usr/bin first which is likely to affect many other things including potentially not finding the perlbrew installation first
-# silences warnings but breaks ifneq '$(VIRTUAL_ENV)$(CONDA_DEFAULT_ENV)$(TRAVIS)' ''
-#
-#ifndef VIRTUAL_ENV
-# VIRTUAL_ENV = ''
-#endif
-#ifndef CONDA_DEFAULT_ENV
-# CONDA_DEFAULT_ENV = ''
-#endif
-#ifndef TRAVIS
-# TRAVIS = ''
-#endif
-
-SUDO := sudo -H
-SUDO_PIP := sudo -H
-
-ifdef VIRTUAL_ENV
- # breaks as command before first target
- #$(info VIRTUAL_ENV environment variable detected, not using sudo)
- SUDO_PIP :=
-endif
-ifdef CONDA_DEFAULT_ENV
- #$(info CONDA_DEFAULT_ENV environment variable detected, not using sudo)
- SUDO_PIP :=
-endif
-#ifdef TRAVIS
- # this breaks before first target
- #$(info TRAVIS environment variable detected, not using sudo)
-# SUDO_PIP :=
-#endif
-
-# must come after to reset SUDO_PIP to blank if root
-# EUID / UID not exported in Make
-# USER not populated in Docker
-ifeq '$(shell id -u)' '0'
- #$(info UID = 0 detected, not using sudo)
- SUDO :=
- SUDO_PIP :=
-endif
-
-PARQUET_VERSION=1.5.0
-
# ===================
# bootstrap commands:
+# setup/bootstrap.sh
+#
+# OR
+#
# Alpine:
#
-# apk add --no-cache git $(MAKE) && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE)
-
+# apk add --no-cache git make && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make
+#
# Debian / Ubuntu:
#
-# apt-get update && apt-get install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE)
-
+# apt-get update && apt-get install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make
+#
# RHEL / CentOS:
#
-# yum install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE)
+# yum install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make
# ===================
+# would fail bootstrapping on Alpine
+#SHELL := /usr/bin/env bash
+
+ifneq ("$(wildcard bash-tools/Makefile.in)", "")
+ include bash-tools/Makefile.in
+endif
+
+REPO := HariSekhon/DevOps-Python-tools
+
+CODE_FILES := $(shell find . -type f -name '*.py' -o -type f -name '*.sh' | grep -v -e bash-tools -e /pylib/)
+
+# placeholders to silence check_makefile.sh warnings - should be set in client Makefiles after sourcing
+ifndef PARQUET_VERSION
+ PARQUET_VERSION :=
+endif
+ifndef SKIP_PARQUET
+ SKIP_PARQUET :=
+endif
+
.PHONY: build
-build:
- @echo =============
+build: init
+ @echo =========================
@echo DevOps Python Tools Build
- @echo =============
+ @echo =========================
+ @$(MAKE) git-summary
+ @echo
+ # defer via external sub-call, otherwise will result in error like
+ # make: *** No rule to make target 'python-version', needed by 'build'. Stop.
+ @$(MAKE) python-version
+
+ if [ -z "$(CPANM)" ]; then make; exit $$?; fi
+ $(MAKE) system-packages-python
+ if type apk 2>/dev/null; then $(MAKE) apk-packages-extra; fi
+ if type apt-get 2>/dev/null; then $(MAKE) apt-packages-extra; fi
- $(MAKE) common
$(MAKE) python
-.PHONY: system-packages
-system-packages:
- if [ -x /sbin/apk ]; then $(MAKE) apk-packages; fi
- if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages; fi
- if [ -x /usr/bin/yum ]; then $(MAKE) yum-packages; fi
- if [ -x /usr/local/bin/brew -a `uname` = Darwin ]; then $(MAKE) homebrew-packages; fi
-
-.PHONY: common
-common: system-packages submodules
- :
-
-.PHONY: submodules
-submodules:
- git submodule init
- git submodule update --recursive
-
-.PHONY: python
-python:
+.PHONY: init
+init:
+ git submodule update --init --recursive
+
+# Magic to build the dependencies for only given program(s)
+#
+# dependency of same % stem prefix checks for a matching .py file to consider it a valid target
+#
+# doesn't work
+#.PHONY: all
+#.PHONY: %.pyc
+# TODO: doesn't work, says nothing to be done even when .pyc isn't present, and allows make anonymize22.py which breaks
+#%.py: %.pyc
+# @$(MAKE) $@c
+%.pyc:: %.py
+ @# this utility script supports taking .pyc or .pyo names and still does the right thing
+ @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \
+ python -m py_compile $< && \
+ echo && \
+ echo Generated $@
+%.pyo:: %.py
+ @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \
+ python -O -m py_compile $< && \
+ echo && \
+ echo Generated $@
+
+.PHONY: pylib
+pylib:
+ @$(MAKE) python-version
cd pylib && $(MAKE)
- # don't pull parquet tools in to docker image by default, will bloat it
- # can fetch separately by running 'make parquet-tools' if you really want to
- if [ -f /.dockerenv -o -n "$(SKIP_PARQUET)" ]; then \
+
+.PHONY: python
+python: pylib
+ # defer via external sub-call, otherwise will result in error like
+ # make: *** No rule to make target 'python-version', needed by 'build'. Stop.
+ @# don't pull parquet tools in to docker image by default, will bloat it
+ @# can fetch separately by running 'make parquet-tools' if you really want to
+ @if [ -f /.dockerenv -o -n "$(SKIP_PARQUET)" ]; then \
echo; echo; \
echo "Skipping Parquet install..."; \
echo; echo; \
else \
$(MAKE) parquet-tools; \
fi
-
- # json module built-in to Python >= 2.6, backport not available via pypi
- #$(SUDO_PIP) pip install json
-
- # for impyla
- $(SUDO_PIP) pip install --upgrade setuptools || :
- #
- # snappy may fail to install on Mac not finding snappy-c.h - workaround:
- #
- # brew install snappy
- #
- # find /usr/local -name snappy-c.h
- #
- # /usr/local/include/snappy-c.h
- #
- # sudo su
- # LD_RUN_PATH=/usr/local/include pip install snappy
- #
- $(SUDO_PIP) pip install --upgrade -r requirements.txt
- # prevents https://urllib3.readthedocs.io/en/latest/security.html#insecureplatformwarning
- $(SUDO_PIP) pip install --upgrade ndg-httpsclient
- # for ipython-notebook-pyspark.py
- #$(SUDO_PIP) pip install jinja2
- # HiveServer2
- #$(SUDO_PIP) pip install pyhs2
- # Impala
- #$(SUDO_PIP) pip install impyla
- # must downgrade happybase library to work on Python 2.6
- if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi
-
- # Python >= 2.7 - won't build on 2.6, handle separately and accept failure
- $(SUDO_PIP) pip install "ipython[notebook]" || :
+
+ @# only install pip packages not installed via system packages
+ @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt
+ @#$(SUDO_PIP) $(PIP) install -r requirements.txt
+ @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt
+
+ @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3
+ @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit
+ PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || :
+
+ # Python >= 3.4 - try but accept failure in case we're not on the right version of Python
+ @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3"; fi
+ PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3" || :
+
+ @# for impyla
+ @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || :
+ @#
+ @# snappy may fail to install on Mac not finding snappy-c.h - workaround:
+ @#
+ @# brew install snappy
+ @#
+ @# find /usr/local -name snappy-c.h
+ @#
+ @# /usr/local/include/snappy-c.h
+ @#
+ @# sudo su
+ @# LD_RUN_PATH=/usr/local/include $(PIP) install snappy
+ @#
+ @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt
+
+ @# for ipython-notebook-pyspark.py
+ @#$(SUDO_PIP) PIP=$(PIP) install jinja2
+ @# HiveServer2
+ @#$(SUDO_PIP) $(PIP) install pyhs2
+ @# Impala
+ @#$(SUDO_PIP) $(PIP) install impyla
+ @# must downgrade happybase library to work on Python 2.6
+ @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi
+
+ @# Python >= 2.7 - won't build on 2.6, handle separately and accept failure
+ @PIP=$(PIP) bash-tools/python/python_pip_install.sh "ipython[notebook]" || :
@echo
- bash-tools/python_compile.sh
+ $(MAKE) pycompile
@echo
@echo
$(MAKE) spark-deps
@echo
@echo 'BUILD SUCCESSFUL (pytools)'
-.PHONY: quick
-quick:
- QUICK=1 $(MAKE)
-
.PHONY: parquet-tools
parquet-tools:
- if ! [ -d "parquet-tools-$(PARQUET_VERSION)" ]; then \
- wget -t 100 --retry-connrefused -c -O "parquet-tools-$(PARQUET_VERSION)-bin.zip" "http://search.maven.org/remotecontent?filepath=com/twitter/parquet-tools/$(PARQUET_VERSION)/parquet-tools-$(PARQUET_VERSION)-bin.zip"; \
- unzip "parquet-tools-$(PARQUET_VERSION)-bin.zip"; \
- fi
+ @BIN='.' bash-tools/install/install_parquet-tools.sh
-.PHONY: apk-packages
-apk-packages:
- $(SUDO) apk update
- $(SUDO) apk add `sed 's/#.*//; /^[[:space:]]*$$/d' setup/apk-packages.txt setup/apk-packages-dev.txt`
+.PHONY: apk-packages-extra
+apk-packages-extra:
if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apk add openjdk8-jre-base; fi
# Spark Java Py4J gets java linking error without this
if [ -f /lib/libc.musl-x86_64.so.1 ]; then [ -e /lib/ld-linux-x86-64.so.2 ] || ln -sv /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2; fi
+.PHONY: apt-packages-extra
+apt-packages-extra:
+ if [ -z "$(NOJAVA)" ]; then which java || bash-tools/packages/apt_install_packages.sh default-jdk; fi
+
# for validate_multimedia.py
# available in Alpine 2.6, 2.7 and 3.x
.PHONY: apk-packages-multimedia
@@ -173,44 +184,13 @@ apk-packages-multimedia:
$(SUDO) apk update
$(SUDO) apk add ffmpeg
-.PHONY: apk-packages-remove
-apk-packages-remove:
- cd pylib && $(MAKE) apk-packages-remove
- $(SUDO) apk del `sed 's/#.*//; /^[[:space:]]*$$/d' < setup/apk-packages-dev.txt` || :
- $(SUDO) rm -fr /var/cache/apk/*
-
-.PHONY: apt-packages
-apt-packages:
- $(SUDO) apt-get update
- $(SUDO) apt-get install -y `sed 's/#.*//; /^[[:space:]]*$$/d' setup/deb-packages.txt setup/deb-packages-dev.txt`
- if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk; fi
-
# for validate_multimedia.py
# Ubuntu 16.04 Xenial onwards, not available in Ubuntu 14.04 Trusty
# Debian 9 Stretch onwards, not available in Debian 8 Jessie
.PHONY: apt-packages-multimedia
apt-packages-multimedia:
- $(SUDO) apt-get update
- $(SUDO) apt-get install -y --no-install-recommends ffmpeg
-
-.PHONY: apt-packages-remove
-apt-packages-remove:
- cd pylib && $(MAKE) apt-packages-remove
- $(SUDO) apt-get purge -y `sed 's/#.*//; /^[[:space:]]*$$/d' < setup/deb-packages-dev.txt`
-
-.PHONY: homebrew-packages
-homebrew-packages:
- # Sudo is not required as running Homebrew as root is extremely dangerous and no longer supported as Homebrew does not drop privileges on installation you would be giving all build scripts full access to your system
- # Fails if any of the packages are already installed, ignore and continue - if it's a problem the latest build steps will fail with missing headers
- brew install `sed 's/#.*//; /^[[:space:]]*$$/d' setup/brew-packages.txt` || :
-
-.PHONY: yum-packages
-yum-packages:
- # python-pip requires EPEL, so try to get the correct EPEL rpm
- rpm -q wget || $(SUDO) yum install -y wget
- rpm -q epel-release || yum install -y epel-release || { wget -t 100 --retry-connrefused -O /tmp/epel.rpm "https://dl.fedoraproject.org/pub/epel/epel-release-latest-`grep -o '[[:digit:]]' /etc/*release | head -n1`.noarch.rpm" && $(SUDO) rpm -ivh /tmp/epel.rpm && rm -f /tmp/epel.rpm; }
- for x in `sed 's/#.*//; /^[[:space:]]*$$/d' setup/rpm-packages.txt setup/rpm-packages-dev.txt`; do rpm -q $$x || $(SUDO) yum install -y $$x; done
- if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) yum install -y java; fi
+ $(SUDO) apt-get update -o DPkg::Lock::Timeout=1200
+ $(SUDO) apt-get install -o DPkg::Lock::Timeout=1200 -y --no-install-recommends ffmpeg
# for validate_multimedia.py
.PHONY: yum-packages-multimedia
@@ -218,68 +198,37 @@ yum-packages-multimedia:
@echo "This requires 3rd party rpm repos which could result in rpm hell, please handle this manually yourself so you understand what you're doing"
exit 1
-.PHONY: yum-packages-remove
-yum-packages-remove:
- cd pylib && $(MAKE) yum-packages-remove
- for x in `sed 's/#.*//; /^[[:space:]]*$$/d' < setup/rpm-packages-dev.txt`; do rpm -q $$x && $(SUDO) yum remove -y $$x; done
-
.PHONY: jython
jython:
- if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi
- if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi
- if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi
+ @#if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi
+ @#if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi
+ @#if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi
+ bash-tools/packages/install_packages.sh wget expect
sh jython_install.sh
-.PHONY: sonar
-sonar:
- sonar-scanner
-
.PHONY: test-lib
test-lib:
cd pylib && $(MAKE) test
.PHONY: test
-test: test-lib
+#test: test-lib
+test:
tests/all.sh
.PHONY: basic-test
basic-test: test-lib
- bash-tools/all.sh
-
-.PHONY: test2
-test2:
- cd pylib && $(MAKE) test2
- tests/all.sh
+ bash-tools/checks/check_all.sh
.PHONY: install
-install:
+install: build
@echo "No installation needed, just add '$(PWD)' to your \$$PATH"
-.PHONY: update
-update: update2 build
- :
-
-.PHONY: update2
-update2: update-no-recompile
- :
-
-.PHONY: update-no-recompile
-update-no-recompile:
- git pull
- git submodule update --init --recursive
-
-.PHONY: update-submodules
-update-submodules:
- git submodule update --init --remote
-.PHONY: updatem
-updatem: update-submodules
- :
-
.PHONY: clean
clean:
cd pylib && $(MAKE) clean
@# the xargs option to ignore blank input doesn't work on Mac
- @find . -maxdepth 3 -iname '*.py[co]' -o -iname '*.jy[co]' | xargs rm -f
+ @find . -iname '*.py[co]' -o -iname '*.jy[co]' | xargs rm -f
+ @find . -iname '*.spec' | xargs rm -f
@find . -type d -ipath '*/tests/*' -iname 'test-*spark*.avro' | xargs rm -rf
@find . -type d -ipath '*/tests/*' -iname 'test-*spark*.parquet' | xargs rm -rf
@rm -f parquet-tools-$(PARQUET_VERSION)-bin.zip
@@ -294,22 +243,15 @@ spark-deps:
rm -vf spark-deps.zip
zip spark-deps.zip pylib
-.PHONY: docker-run
-docker-run:
- docker run -ti --rm harisekhon/pytools ${ARGS}
-
-.PHONY: run
-run:
- $(MAKE) docker-run
-
-.PHONY: docker-mount
-docker-mount:
- docker run -ti --rm -v $$PWD:/py harisekhon/pytools bash -c "cd /py; exec bash"
-
-.PHONY: mount
-mount: docker-mount
- :
-
-.PHONY: push
-push:
- git push
+.PHONY: dockerhub-trigger
+dockerhub-trigger:
+ # PyTools
+ curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/d470810b-9a44-4abc-92cc-c903a6afd962/trigger/0e69c39f-ea1b-43c7-a97d-cef1252f1400/call/
+ # Alpine Github
+ curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/df816f2a-9407-4f1b-8b51-39615d784e65/trigger/8d9cb826-48df-439c-8c20-1975713064fc/call/
+ # Debian Github
+ curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/439eff84-50c7-464a-a49e-0ac0bf1a9a43/trigger/0cfb3fe7-2028-494b-a43b-068435e6a2b3/call/
+ # CentOS Github
+ curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/efba1846-5a9e-470a-92f8-69edc1232ba0/trigger/316d1158-7ffb-49a4-a7bd-8e5456ba2d15/call/
+ # Ubuntu Github
+ curl --header "Content:Type:application/json" --data '{"build":true}' -X POST https://cloud.docker.com/api/build/v1/source/8b3dc094-d4ca-4c92-861e-1e842b5fac42/trigger/abd4dbf0-14bc-454f-9cde-081ec014bc48/call/
diff --git a/README.md b/README.md
index e9f448a4a..5e1fe9bca 100644
--- a/README.md
+++ b/README.md
@@ -1,175 +1,432 @@
-Hari Sekhon - DevOps Python Tools
-=================================
-[](https://travis-ci.org/HariSekhon/devops-python-tools)
-[](https://www.codacy.com/app/harisekhon/devops-python-tools)
-[](https://github.com/harisekhon/devops-python-tools/stargazers)
-[](https://github.com/harisekhon/devops-python-tools/network)
-[](https://gemnasium.com/github.com/HariSekhon/devops-python-tools)
-[](https://github.com/harisekhon/devops-python-tools#hari-sekhon-pytools)
-[](https://hub.docker.com/r/harisekhon/pytools/)
-[](http://microbadger.com/#/images/harisekhon/pytools)
-
-### Hadoop, Spark / PySpark, HBase, Pig, Ambari, IPython and Linux Tools ###
-
-A few of the Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options.
-
-For many more tools see the [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) and [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) repos which contains many Hadoop, NoSQL, Web and infrastructure tools and Nagios plugins.
+# Hari Sekhon - DevOps Python Tools
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/stargazers)
+[](https://github.com/HariSekhon/DevOps-Python-tools/network)
+[](https://github.com/boyter/scc/)
+[](https://github.com/boyter/scc/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE)
+[](https://www.linkedin.com/in/HariSekhon/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/commits/master)
+
+
+
+
+[](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard)
+[](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://hub.docker.com/r/harisekhon/github/)
+[](https://github.com/HariSekhon/Dockerfiles)
+[](https://hub.docker.com/r/harisekhon/pytools)
+[](https://hub.docker.com/r/harisekhon/pytools/)
+[](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,Dockerfiles&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,HAProxy-configs&r=HariSekhon,SQL-scripts)
+[](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STARCHARTS.md)
+
+
+[](https://harisekhon.github.io/CI-CD/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.concourse.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.gocd.yml)
+[](https://github.com/HariSekhon/TeamCity-CI)
+
+[](https://circleci.com/gh/HariSekhon/DevOps-Python-tools)
+[](https://buildkite.com/hari-sekhon/devops-python-tools)
+[](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master)
+[](https://cloud.drone.io/HariSekhon/DevOps-Python-tools)
+[](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week)
+[](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools)
+[](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml)
+
+[](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master)
+[](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines)
+[](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://gitlab.com/HariSekhon/DevOps-Python-tools)
+[](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools)
+[](https://bitbucket.org/HariSekhon/DevOps-Python-tools)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+22.04%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+12%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Fedora%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22)
+
+[git.io/pytools](https://git.io/pytools)
+
+## AWS, Docker, Spark, Hadoop, HBase, Hive, Impala, Python & Linux Tools
+
+DevOps, Cloud, Big Data, NoSQL, Python & Linux tools. All programs have `--help`.
Hari Sekhon
-Big Data Contractor, United Kingdom
+Cloud & Big Data Contractor, United Kingdom
-https://www.linkedin.com/in/harisekhon
+[](https://www.linkedin.com/in/HariSekhon/)
+
*(you're welcome to connect with me on LinkedIn)*
-##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. #####
+**Make sure you run `make update` if updating and not just `git pull` as you will often need the latest library submodule and possibly new upstream libraries**
-### Quick Start ###
+## Quick Start
-#### Ready to run Docker image #####
+### Ready to run Docker image
All programs and their pre-compiled dependencies can be found ready to run on [DockerHub](https://hub.docker.com/r/harisekhon/pytools/).
List all programs:
-```
+
+```shell
docker run harisekhon/pytools
```
+
Run any given program:
-```
+
+```shell
docker run harisekhon/pytools
```
-#### Automated Build from source #####
+### Automated Build from source
+installs git, make, pulls the repo and build the dependencies:
+
+```shell
+curl -L https://git.io/python-bootstrap | sh
```
-git clone https://github.com/harisekhon/devops-python-tools pytools
+
+or manually:
+
+```shell
+git clone https://github.com/HariSekhon/DevOps-Python-tools pytools
cd pytools
make
```
-Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/devops-python-tools#detailed-build-instructions) further down for more information.
-
-Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/harisekhon/devops-python-tools#jython-for-hadoop-utils) for details.
-
-### Usage ###
-
-All programs come with a ```--help``` switch which includes a program description and the list of command line options.
-
-Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```.
-
-### DevOps Python Tools ###
-
-- [Spark](https://spark.apache.org/) & Data Format Converters:
- - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter
- - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter
- - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas
- - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas
- - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter
- - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter
- - ```json_to_xml.py``` - JSON to XML converter
- - ```xml_to_json.py``` - XML to JSON converter
- - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json'
- - see also ```validate_*.py``` further down for all these formats and more
-- ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename
-- [Ambari](https://hortonworks.com/apache/ambari/):
- - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API
- - list blueprints
- - fetch all blueprints or a specific blueprint to local json files
- - blueprint an existing cluster
- - create a new cluster using a blueprint
- - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control
- - optionally strips out the excessive and overly specific configs to create generic more reusable templates
- - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool
- - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts
- - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API
- - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API
-- [Hadoop](http://hadoop.apache.org/) HDFS:
- - ```hadoop_hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports.
- - ```hadoop_hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum)
- - ```hadoop_hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files
-- [HBase](https://hbase.apache.org/):
- - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written.
- - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables
- - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions
- - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix
- - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables.
- - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables.
- - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending
- - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting
- - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting
- - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold)
-- [OpenTSDB](http://opentsdb.net/):
- - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting
- - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending
-- [Pig](https://pig.apache.org/)
- - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch)
- - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud)
- - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop
-- ```ipython-notebook-pyspark.py``` - per-user authenticated IPython Notebook + PySpark integration to allow each user to auto-create their own password protected IPython Notebook running Spark
+To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension
+instead of `.py`:
+
+```shell
+make anonymize.pyc
+```
+
+Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/DevOps-Python-tools#detailed-build-instructions) further down for more information.
+
+Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/HariSekhon/DevOps-Python-tools#jython-for-hadoop-utils) for details.
+
+### Usage
+
+All programs come with a `--help` switch which includes a program description and the list of command line options.
+
+Environment variables are supported for convenience and also to hide credentials from being exposed in the process list
+eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and
+often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`.
+
+### DevOps Python Tools - Inventory
+
+- Linux:
+ - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing
+ - lists)
+ - anonymizations include these and more:
+ - hostnames / domains / FQDNs
+ - email addresses
+ - IP + MAC addresses
+ - AWS Access Keys, Secret Keys, ARNs, STS tokens
+ - Kerberos principals
+ - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...)
+ - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings
+ - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to ``
+ - placeholder tokens indicate what was stripped out (eg. ``, ``, ``)
+ - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes
+ communicating with each other to compare configs and log communications
+ - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support
+ teams can differentiate hosts in clusters
+ - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel
+ before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation
+ important for anonymization rules, as well as maintaining file content order. On servers this parallelization can
+ result in a 30x speed up for large log files
+ - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file
+ basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename
+ - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments,
+ useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc).
+ Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See
+ further down for more details and sub-programs that simplify usage for many of the most common cluster technologies
+ - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in
+ your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo)
+- [Amazon Web Services](https://aws.amazon.com/):
+ - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally
+ filters for active keys and older than N days (for key rotation governance)
+ - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have
+ never been used (these should generally be removed/disabled). Optionally filters for only active keys
+ - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys.
+ Optionally filters for users not used in the last N days to find old accounts to remove
+ - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for
+ users with passwords not used in the last N days
+- [Google Cloud Platform](https://cloud.google.com/):
+ - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python:
+ - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to
+ [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is
+ triggered by [Cloud Scheduler](https://cloud.google.com/scheduler)
+ - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and
+ [Cloud Scheduler](https://cloud.google.com/scheduler) jobs
+ - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP
+ address - use this to test your VPC connector public routing, comparison with firewall rules etc.
+ - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its
+ accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via
+ VPC connector routing
+ - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with
+ their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days
- [Docker](https://www.docker.com/):
- - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh```
- - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (official `docker search` is limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh```
- - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION```
-- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch)
- - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required
- - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA
- - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA
- - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA
- - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing)
- - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing)
- - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node
- - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore
- - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator
- - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server
- - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node
- - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node
- - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins)
-- ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red
-- ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI
-- ```validate_*.py``` - validate files, directory trees and/or standard input streams
- - supports the following file formats:
- - Avro
- - CSV
- - INI / Java Properties (also detects duplicate sections and duplicate keys within sections)
- - JSON (both normal and json-doc-per-line bulk / big data format as found in MongoDB and Hadoop json data files)
- - LDAP LDIF
- - Parquet
- - XML
- - YAML
- - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`)
- - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies
-- ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo)
+ - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a
+ docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very
+ useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only
+ the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests
+ across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh`
+ - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official
+ `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned
+ to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy
+ shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg.
+ `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all
+ docker images eg. `docker_pull_all_images_all_tags.sh`
+ - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION`
+- [Spark](https://spark.apache.org/) & Data Format Converters:
+ - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter
+ - `spark_parquet_to_avro.py` - PySpark Parquet => Avro converter
+ - `spark_csv_to_avro.py` - PySpark CSV => Avro converter, supports both inferred and explicit schemas
+ - `spark_csv_to_parquet.py` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas
+ - `spark_json_to_avro.py` - PySpark JSON => Avro converter
+ - `spark_json_to_parquet.py` - PySpark JSON => Parquet converter
+ - `xml_to_json.py` - XML to JSON converter
+ - `json_to_xml.py` - JSON to XML converter
+ - `json_to_yaml.py` - JSON to YAML converter
+ - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for
+ pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and
+ [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json /
+ directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard
+ output for convenient command line chaining and redirection, optionally continues on error, collects broken records
+ to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not
+ technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json'
+ - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON)
+ - see also `validate_*.py` further down for all these formats and more
+- [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL:
+ - [Ambari](https://hortonworks.com/apache/ambari/):
+ - `ambari_blueprints.py` - Blueprint cluster templating and deployment tool using Ambari API
+ - list blueprints
+ - fetch all blueprints or a specific blueprint to local json files
+ - blueprint an existing cluster
+ - create a new cluster using a blueprint
+ - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for
+ proper revision control
+ - optionally strips out the excessive and overly specific configs to create generic more reusable templates
+ - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable
+ using this tool
+ - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts
+ - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API
+ - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API
+ - [Hadoop](http://hadoop.apache.org/) HDFS:
+ - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to
+ replication factor 3 to avoid missing block alerts during datanode maintenance windows
+ - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file
+ or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data
+ layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports.
+ - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster
+ than doing `hdfs dfs -cat | md5sum`)
+ - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree
+ showing block size, replication factor, underfilled blocks and small files
+ - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/):
+ - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format
+ to standard output
+
+ The following programs can all optionally filter by database / table name regex:
+
+ - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala
+ table
+ - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation
+ between cluster migrations
+ - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for
+ finding unusually wide tables
+ - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts.
+ Useful for finding unusually big tables
+ - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where
+ any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or
+ subtle ETL bugs
+ - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs.
+ Useful for catching data quality problems or subtle ETL bugs
+ - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain
+ NULLs. Useful for catching data quality problems or subtle ETL bugs
+ - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL
+ property from describe table
+ - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location
+ - [HBase](https://hbase.apache.org/):
+ - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table,
+ with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI
+ tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written.
+ - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting
+ tables
+ - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table,
+ giving per region row counts and % of total rows for the table as well as median and quartile row counts per
+ regions
+ - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in
+ an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row
+ counts per prefix
+ - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating
+ on all tables or takes an optional regex and compacts only matching tables.
+ - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an
+ optional regex and flushes only matching tables.
+ - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or
+ memStoreSize, ascending or descending
+ - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average
+ since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes /
+ total requests per second. Useful for watching more granular region stats to detect region hotspotting
+ - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or
+ average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read,
+ write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular
+ RegionServer stats to detect RegionServer hotspotting
+ - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received
+ the least requests (requests below a given threshold)
+ - [OpenTSDB](http://opentsdb.net/):
+ - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew
+ and help avoid HBase region hotspotting
+ - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase
+ tables with optionally their created date, sorted ascending
+ - [Pig](https://pig.apache.org/)
+ - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to
+ [Elasticsearch](https://www.elastic.co/products/elasticsearch)
+ - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to
+ [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud)
+ - `pig_udfs.jy` - Pig Jython UDFs for Hadoop
+- `find_active_server.py` - returns first available healthy server or active master in high availability deployments,
+ useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex
+ content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for
+ which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you
+ want to query cluster wide information available from any online peer (eg. Elasticsearch)
+ - The following are simplified specialisations of the above program, just pass host arguments, all the details have
+ been baked in, no switches required
+ - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA
+ - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA
+ - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA
+ - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run
+ multiple of these for load balancing)
+ - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server
+ (run multiple of these for load balancing)
+ - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node
+ - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node
+ - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad,
+ Catalog or Statestore
+ - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator
+ - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server
+ - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server
+ - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node
+ - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node
+ - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the
+ [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins)
+- [Travis CI](https://travis-ci.org/):
+ - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo -
+ useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red
+ - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks
+ session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot
+ debug launcher for Travis CI
+- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as
+ Chrome and Firefox to fetch a given URL and content/regex match the result
+- Data Validation (useful in CI):
+ - `validate_*.py` - validate files, directory trees and/or standard input streams
+ - supports the following file formats:
+ - Avro
+ - CSV
+ - INI / Java Properties (also detects duplicate sections and duplicate keys within sections)
+ - JSON (both normal and json-doc-per-line bulk / big data format as found in MongoDB and Hadoop json data files)
+ - LDAP LDIF
+ - Parquet
+ - XML
+ - YAML
+ - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`,
+ `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`)
+ - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for
+ things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images
+ [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data &
+ Linux technologies
### Detailed Build Instructions
-##### Python VirtualEnv localized installs
+#### Python VirtualEnv localized installs
-The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages instead of installing from PyPI then follow the Manual Build section below.
+The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it
+detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages
+instead of installing from PyPI then follow the Manual Build section below.
-
-#### Manual Setup
+### Manual Setup
Enter the pytools directory and run git submodule init and git submodule update to fetch my library repo:
-```
-git clone https://github.com/harisekhon/devops-python-tools pytools
+```shell
+git clone https://github.com/HariSekhon/DevOps-Python-tools pytools
cd pytools
git submodule init
git submodule update
sudo pip install -r requirements.txt
```
-
-#### Offline Setup
+### Offline Setup
Download the DevOps Python Tools and Pylib git repos as zip files:
-https://github.com/HariSekhon/devops-python-tools/archive/master.zip
+
-https://github.com/HariSekhon/pylib/archive/master.zip
+
-Unzip both and move Pylib to the ```pylib``` folder under DevOps Python Tools.
+Unzip both and move Pylib to the `pylib` folder under DevOps Python Tools.
-```
+```shell
unzip devops-python-tools-master.zip
unzip pylib-master.zip
@@ -178,137 +435,246 @@ mv -v pylib-master pylib
mv -vf pylib pytools/
```
-Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions).
+Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal
+mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions).
-All PyPI modules are listed in the ```requirements.txt``` file.
+All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files.
Internal Mirror example ([JFrog Artifactory](https://jfrog.com/artifactory/) or similar):
-```
+```shell
sudo pip install --index https://host.domain.com/api/pypi/repo/simple --trusted host.domain.com -r requirements.txt
```
Proxy example:
-```
+```shell
sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt
```
-##### Mac OS X
+#### Mac OS X
+
+The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing
+`git` is enough to trigger Xcode install).
-The automated build also works on Mac OS X but you'll need to download and install [Apple XCode](https://developer.apple.com/download/). I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X):
+I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for
+development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X):
+```shell
+bash-tools/install/install_homebrew.sh
```
+
+```shell
brew install openssl wget
```
If failing to build an OpenSSL lib dependency, just prefix the build command like so:
-```
+```shell
sudo OPENSSL_INCLUDE=/usr/local/opt/openssl/include OPENSSL_LIB=/usr/local/opt/openssl/lib ...
```
-You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is caused by pip 10 vs pip 9 and downgrading will work around it:
+You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is
+caused by pip 10 vs pip 9 and downgrading will work around it:
-```
+```shell
sudo pip install --upgrade pip==9.0.1
make
sudo pip install --upgrade pip
make
```
-### Jython for Hadoop Utils ###
+### Jython for Hadoop Utils
The 3 Hadoop utility programs listed below require Jython (as well as Hadoop to be installed and correctly configured)
-```
-hadoop_hdfs_time_block_reads.jy
-hadoop_hdfs_files_native_checksums.jy
-hadoop_hdfs_files_stats.jy
+```shell
+hdfs_time_block_reads.jy
+hdfs_files_native_checksums.jy
+hdfs_files_stats.jy
```
Run like so:
-```
-jython -J-cp $(hadoop classpath) hadoop_hdfs_time_block_reads.jy --help
+
+```shell
+jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help
```
-The ```-J-cp $(hadoop classpath) ``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs.
+The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop
+APIs.
See below for procedure to install Jython if you don't already have it.
-##### Automated Jython Install
+#### Automated Jython Install
This will download and install jython to /opt/jython-2.7.0:
-```
+```shell
make jython
```
-##### Manual Jython Install
+#### Manual Jython Install
-Jython is a simple download and unpack and can be fetched from http://www.jython.org/downloads.html
+Jython is a simple download and unpack and can be fetched from
Then add the Jython install bin directory to the $PATH or specify the full path to the `jython` binary, eg:
-```
-/opt/jython-2.7.0/bin/jython hadoop_hdfs_time_block_reads.jy ...
+```shell
+/opt/jython-2.7.0/bin/jython hdfs_time_block_reads.jy ...
```
+### Configuration for Strict Domain / FQDN validation
-#### Configuration for Strict Domain / FQDN validation ####
+Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my
+[PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`,
+`.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal
+environments).
-Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/harisekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like ```.local``` or ```.intranet``` (both supported by default).
-
-#### Python SSL certificate verification problems
+### Python SSL certificate verification problems
If you end up with an error like:
-```
+
+```shell
./dockerhub_show_tags.py centos ubuntu
[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:765)
```
-It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One quick fix is to do the following:
-```
+
+It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One
+quick fix is to do the following:
+
+```shell
sudo pip uninstall -y certifi &&
sudo pip install certifi==2015.04.28
```
-### Updating ###
+### Updating
+
+Run:
+
+```shell
+make update
+```
-Run ```make update```. This will git pull and then git submodule update which is necessary to pick up corresponding library updates.
+This will git pull and then git submodule update which is necessary to pick up corresponding library updates.
-If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run ```make update-no-recompile``` (will miss new library dependencies - do full ```make update``` if you encounter issues).
+If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each
+time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter
+issues).
### Testing
[Continuous Integration](https://travis-ci.org/HariSekhon/devops-python-tools) is run on this repo with tests for success and failure scenarios:
-- unit tests for the custom supporting [python library](https://github.com/harisekhon/pylib)
+
+- unit tests for the custom supporting [python library](https://github.com/HariSekhon/pylib)
- integration tests of the top level programs using the libraries for things like option parsing
-- [functional tests](https://github.com/HariSekhon/devops-python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/)
+- [functional tests](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/)
To trigger all tests run:
-```
+```shell
make test
```
-which will start with the underlying libraries, then move on to top level integration tests and functional tests using docker containers if docker is available.
+which will start with the underlying libraries, then move on to top level integration tests and functional tests using
+docker containers if docker is available.
-### Contributions ###
+### Contributions
Patches, improvements and even general feedback are welcome in the form of GitHub pull requests and issue tickets.
-### See Also ###
+You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another
+Hortonworks guy Jonas Straub:
+
+
+
+## Star History
+
+[](https://star-history.com/#HariSekhon/DevOps-Python-tools&Date)
+
+[git.io/python-tools](https://git.io/python-tools)
+
+[git.io/pytools](https://git.io/pytools)
+
+## More Core Repos
+
+
+
+### Knowledge
+
+[](https://github.com/HariSekhon/Knowledge-Base)
+[](https://github.com/HariSekhon/Diagrams-as-Code)
+
+
+
+### DevOps Code
+
+[](https://github.com/HariSekhon/DevOps-Bash-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Perl-tools)
+[](https://github.com/HariSekhon/DevOps-Golang-tools)
+
+
+
+### Containerization
+
+[](https://github.com/HariSekhon/Kubernetes-configs)
+[](https://github.com/HariSekhon/Dockerfiles)
+
+### CI/CD
+
+[](https://github.com/HariSekhon/GitHub-Actions)
+[](https://github.com/HariSekhon/Jenkins)
+
+### Databases - DBA - SQL
+
+[](https://github.com/HariSekhon/SQL-scripts)
+
+### DevOps Reloaded
+
+[](https://github.com/HariSekhon/HAProxy-configs)
+[](https://github.com/HariSekhon/Terraform)
+[](https://github.com/HariSekhon/Packer)
+[](https://github.com/HariSekhon/Ansible)
+[](https://github.com/HariSekhon/Environments)
+
+### Monitoring
+
+[](https://github.com/HariSekhon/Nagios-Plugins)
+[](https://github.com/HariSekhon/Nagios-Plugin-Kafka)
+[](https://github.com/HariSekhon/Prometheus)
+
+### Templates
+
+[](https://github.com/HariSekhon/Templates)
+[](https://github.com/HariSekhon/Template-repo)
-* [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 30+ tools for Hadoop, NoSQL, Solr, Elasticsearch, Pig, Hive, Web URL + Nginx stats watchers, SQL and NoSQL syntax recasers, various Linux CLI tools
+### Desktop
-* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 400+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux)
+[](https://github.com/HariSekhon/TamperMonkey)
+[](https://github.com/HariSekhon/Hammerspoon)
+[](https://github.com/HariSekhon/MPV-Scripts)
-* [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule
+### Spotify
-* [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library
+[](https://github.com/HariSekhon/Spotify-tools)
+[](https://github.com/HariSekhon/Spotify-playlists)
-* [Spark Apps eg. Spark => Elasticsearch](https://github.com/harisekhon/spark-to-elasticsearch) - Scala application to index from Spark to Elasticsearch. Used to index data in Hadoop clusters or local data via Spark standalone. This started as a Scala Spark port of ```pig-text-to-elasticsearch.pig``` from this repo.
+The rest of my original source repos are
+[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers).
-You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub:
+Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/)
+and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo.
-* https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb
+
diff --git a/ambari_ams_metrics.sh b/ambari_ams_metrics.sh
index 0c9e92a65..64f20f0a2 100755
--- a/ambari_ams_metrics.sh
+++ b/ambari_ams_metrics.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-16 22:14:34 +0100 (Mon, 16 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# There were 2346 metrics last I checked but this probably varies a lot based on what services are deployed
@@ -98,13 +98,14 @@ if [ -z "$ams_host" ]; then
elif [ -z "$ams_port" ]; then
usage "--port not defined"
fi
-if [ -n "$node_list" -a -z "$metric" ]; then
+if [ -n "$node_list" ] && [ -z "$metric" ]; then
usage "--nodes option requires --metric. See --list-metrics for a list of available metrics"
fi
check_bin(){
local bin="$1"
- if ! which $bin &>/dev/null; then
+ # shellcheck disable=SC2230
+ if ! type -P "$bin" &>/dev/null; then
echo "'$bin' command not found in \$PATH ($PATH)"
exit 1
fi
@@ -125,7 +126,7 @@ list_metrics(){
list_hosts(){
curl -s "$ams_host:$ams_port/ws/v1/timeline/metrics/hosts" |
python -m json.tool |
- egrep '^[[:space:]]*".+":[[:space:]]* \[[[:space:]]*$' |
+ grep -E '^[[:space:]]*".+":[[:space:]]* \[[[:space:]]*$' |
sed 's/"//g;s/:.*//;s/[[:space:]]*//g' |
grep -v fakehostname |
sort ||
diff --git a/ambari_blueprints.py b/ambari_blueprints.py
index 643bc3374..e5dd52217 100755
--- a/ambari_blueprints.py
+++ b/ambari_blueprints.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-08 14:09:50 +0000 (Sun, 08 Nov 2015)
# (re-instantiated from a Perl version in 2014)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -90,7 +90,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.10.2'
+__version__ = '0.10.3'
class AmbariBlueprintTool(CLI):
@@ -116,7 +116,7 @@ def connection(self, host, port, user, password, ssl=False, **kwargs):
#log.info("contacting Ambari as '%s'" % self.user)
if not isHost(host) or not isPort(port) or not isUser(user) or not password:
raise InvalidOptionException('invalid options passed to AmbariBlueprint()')
- proto = 'http' # pylint: disable=unused-variable
+ proto = 'http' # pylint: disable=unused-variable,possibly-unused-variable
if ssl:
proto = 'https'
self.host = host
@@ -140,8 +140,8 @@ def connection(self, host, port, user, password, ssl=False, **kwargs):
except IOError as _:
die("'failed to create dir '%s': %s" % (self.blueprint_dir, _))
- # TODO: change to @staticmethod
- def parse_cluster_name(self, item): # pylint: disable=no-self-use
+ @staticmethod
+ def parse_cluster_name(item):
if isStr(item):
item = json.loads(item)
try:
@@ -154,8 +154,8 @@ def get_clusters(self):
json_data = self.list('clusters')
return [self.parse_cluster_name(item) for item in json_data['items']]
- # TODO: change to @staticmethod
- def parse_blueprint_name(self, item): # pylint: disable=no-self-use
+ @staticmethod
+ def parse_blueprint_name(item):
if isStr(item):
item = json.loads(item)
try:
@@ -168,8 +168,8 @@ def get_blueprints(self):
json_data = self.list('blueprints')
return [self.parse_blueprint_name(item) for item in json_data['items']]
- # TODO: change to @staticmethod
- def parse_host_name(self, item): # pylint: disable=no-self-use
+ @staticmethod
+ def parse_host_name(item):
if isStr(item):
item = json.loads(item)
try:
@@ -320,7 +320,7 @@ def send_blueprint_file(self, filename, name=''):
except KeyError as _:
pass
if not name:
- name = os.path.splitext(os.path.basename(file))[0]
+ name = os.path.splitext(os.path.basename(filename))[0]
log.info("name not specified and couldn't determine blueprint name from blueprint data, reverting to using filename without extension '%s'" % name) # pylint: disable=line-too-long
# this solves the issue of having duplicate Blueprint.blueprint_name keys
try:
@@ -329,7 +329,7 @@ def send_blueprint_file(self, filename, name=''):
data = json.dumps(json_data)
log.info("reset blueprint field name to '%s'" % name)
except ValueError as _:
- qquit('CRITICAL', "invalid json found in file '%s': %s" % (file, name))
+ qquit('CRITICAL', "invalid json found in file '%s': %s" % (filename, name))
except KeyError as _:
log.warn('failed to reset the Blueprint name: %s' % _)
return self.send_blueprint(name, data)
@@ -399,8 +399,8 @@ def save_cluster(self, cluster, path=''):
log.debug("cluster '%s' blueprint content = '%s'" % (cluster, data))
self.save(cluster, path, data)
- # TODO: change to @staticmethod
- def save(self, name, path, data): # pylint: disable=no-self-use
+ @staticmethod
+ def save(name, path, data):
# log.debug('save(%s, %s)' % (name, data))
if data is None:
err = "blueprint '%s' returned None" % name
diff --git a/ambari_cancel_all_requests.sh b/ambari_cancel_all_requests.sh
index 667f0c80e..7d2df4425 100755
--- a/ambari_cancel_all_requests.sh
+++ b/ambari_cancel_all_requests.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-09-27 17:25:36 +0100 (Tue, 27 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -51,10 +51,11 @@ if [ $# -gt 5 ]; then
fi
echo "querying Ambari for request IDs"
-curl -u "$AMBARI_USER:$AMBARI_PASSWORD" "http://$AMBARI_HOST:$AMBARI_PORT/api/v1/clusters/$AMBARI_CLUSTER/requests" |
+curl -u "$AMBARI_USER:$AMBARI_PASSWORD" "$PROTOCOL://$AMBARI_HOST:$AMBARI_PORT/api/v1/clusters/$AMBARI_CLUSTER/requests" |
grep id |
awk '{print $3}' |
-while read id; do
+while read -r id; do
echo "requesting cancellation of request $id"
+ # Unfortunately this call returns 200 OK even if your user account doesn't have permission to do this operation, but will fail on the requests query above for wrong username/password
curl -u "$AMBARI_USER:$AMBARI_PASSWORD" -i -H "X-Requested-By: $AMBARI_USER ($USER)" -X PUT -d '{"Requests":{"request_status":"ABORTED","abort_reason":"Aborted by user"}}' "$PROTOCOL://$AMBARI_HOST:$AMBARI_PORT/api/v1/clusters/$AMBARI_CLUSTER/requests/$id"
done
diff --git a/ambari_trigger_service_checks.py b/ambari_trigger_service_checks.py
index b22006408..2afe64207 100755
--- a/ambari_trigger_service_checks.py
+++ b/ambari_trigger_service_checks.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-23 15:45:28 +0200 (Fri, 23 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -58,7 +58,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.2'
+__version__ = '0.2.2'
class AmbariTriggerServiceChecks(CLI):
@@ -287,8 +287,9 @@ def request_service_checks(self, services):
request_schedule_id = _['resources'][0]['RequestSchedule']['id']
log.info('RequestSchedule %s submitted', request_schedule_id)
href = _['resources'][0]['href']
- assert href == self.url_base.rstrip('/') + '/clusters/{0}/request_schedules/{1}'\
- .format(self.cluster, request_schedule_id)
+ if href != self.url_base.rstrip('/') + '/clusters/{0}/request_schedules/{1}'\
+ .format(self.cluster, request_schedule_id):
+ raise ValueError('href does not match expected request_schedules URI!')
if self.watch:
self.watch_scheduled_request(request_schedule_id)
except (KeyError, ValueError) as _:
@@ -306,7 +307,7 @@ def watch_scheduled_request(self, request_schedule_id):
def parse_scheduled_request(content):
try:
_ = json.loads(content)
- if _['RequestSchedule']['last_execution_status'] == 'COMPLETED':
+ if _['RequestSchedule']['status'] == 'COMPLETED':
log.info('COMPLETED')
return 'COMPLETED'
for item in _['RequestSchedule']['batch']['batch_requests']:
diff --git a/anonymize.py b/anonymize.py
index 73d981e5e..ebcb20ab0 100755
--- a/anonymize.py
+++ b/anonymize.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
@@ -6,16 +6,16 @@
# Date: 2018-08-08 19:02:02 +0100 (Wed, 08 Aug 2018)
# Original Date: 2013-07-18 21:17:41 +0100 (Thu, 18 Jul 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
-# ported from Perl version from DevOps Perl Tools repo (https://github.com/harisekhon/devops-perl-tools)
+# ported from Perl version from DevOps Perl Tools repo (https://github.com/HariSekhon/DevOps-Perl-tools)
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
@@ -36,7 +36,7 @@
Ignore phrases are in a similar file anonymize_ignore.conf, also adjacent to this program.
-Based on Perl Anonymize.pl from https://github.com/harisekhon/devops-perl-tools
+Based on Perl Anonymize.pl from https://github.com/HariSekhon/DevOps-Perl-tools
The Perl version is incredibly faster than Python due to the better regex engine
@@ -61,6 +61,7 @@
from harisekhon.utils import \
isJavaException, \
isPythonTraceback, \
+ isPythonMinVersion, \
isRegex, \
isStr, \
log, \
@@ -71,26 +72,33 @@
# pylint: disable=unused-import
from harisekhon.utils import \
aws_host_ip_regex, \
- domain_regex, \
domain_regex_strict, \
- email_regex, \
filename_regex, \
fqdn_regex, \
host_regex, \
hostname_regex, \
ip_prefix_regex, \
ip_regex, \
- mac_regex, \
subnet_mask_regex, \
user_regex
+ # used dynamically
+ # pylint: disable=unused-import
+ # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend
+ from harisekhon.utils import \
+ domain_regex, \
+ email_regex, \
+ mac_regex \
+ # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend
from harisekhon import CLI
except ImportError as _:
print(traceback.format_exc(), end='')
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.8.3'
+__version__ = '0.11.0'
+ip_regex = r'(?!127\.0\.0\.)' + ip_regex
+subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex
class Anonymize(CLI):
@@ -111,34 +119,40 @@ def __init__(self):
self.hash_salt = None
# order of iteration of application matters because we must do more specific matches before less specific ones
self.anonymizations = OrderedDict([
- ('ip', False),
('ip_prefix', False),
+ ('ip', False),
('subnet_mask', False),
('mac', False),
+ ('db', False),
+ ('generic', False),
+ # kerberos must be applied before email
+ # - if email is applied first, 'user/host@realm' becomes 'user/', exposing user
('kerberos', False),
('email', False),
('password', False),
('ldap', False),
('user', False),
('group', False),
- ('proxy', False),
('http_auth', False),
('cisco', False),
('screenos', False),
('junos', False),
('network', False),
+ ('windows', False),
+ ('aws', False), # access key, secret key, sts tokens etc are very generic so do them later
('fqdn', False),
('domain', False),
('hostname', False),
- ('windows', False),
+ #('proxy', False),
('custom', False),
])
self.exceptions = {
'java_exceptions': False,
'python_tracebacks': False,
}
- self.negative_host_lookbehind = r'(?. anyway by later fqdn anonymization
+ #'aws10': r'ec2-\d+-\d+-\d+-\d+\.{region}(\.compute\.amazonaws\.com)'.format(region='[A-Za-z0-9-]+'),
+ 'db': r'({switch_prefix}(?:db|database)-?name{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ switch_prefix=switch_prefix),
+ 'db2': r'({switch_prefix}(?:db|database)-?instance{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'db3': r'({switch_prefix}schema{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'db4': r'(\s(?:in|of|for)\s+(column|table|database|schema)[\s:]+[\'"])[^\'"]+',
+ 'db5': r'/+user/+hive/+warehouse/+([A-Za-z0-9_-]+/+)*[A-Za-z0-9_-]+.db/+[A-Za-z0-9_-]+',
+ 'generic': r'(\bfileb?)://{filename_regex}'.format(filename_regex=filename_regex),
+ 'generic2': r'({switch_prefix}key{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'generic3': r'({switch_prefix}cluster{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'generic4': r'({switch_prefix}function{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
+ 'generic5': r'({switch_prefix}load-?balancer{id_or_name}?{arg_sep})\S+'\
+ .format(arg_sep=arg_sep,
+ id_or_name=id_or_name,
+ switch_prefix=switch_prefix),
# don't change hostname or fqdn regex without updating hash_hostnames() option parse
# since that replaces these replacements and needs to match the grouping captures and surrounding format
'hostname2': r'({aws_host_ip})(?!-\d)'.format(aws_host_ip=aws_host_ip_regex),
@@ -286,19 +351,29 @@ def __init__(self):
# no \bhost - want to match KrbHost
'hostname5': r'(-host(?:name)?{sep}|host(?:name)?\s*=\s*)({hostname})'\
.format(sep=arg_sep, hostname=hostname_regex),
+ 'hostname6': r'(["\']host(?:name)?["\']\s*:\s*["\']?){hostname}'.format(hostname=hostname_regex),
# use more permissive hostname_regex to catch Kerberos @REALM etc
'domain2': '@{host}'.format(host=hostname_regex),
'group': r'(-{group_name}{sep})\S+'.format(group_name=group_name, sep=arg_sep),
'group2': r'({group_name}{sep}){user}'.format(group_name=group_name, sep=arg_sep, user=user_regex),
'group3': r'for\s+group\s+{group}'.format(group=user_regex),
- 'user': r'(-{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep),
- 'user2': r'/home/{user}'.format(user=user_regex),
+ 'group4': r'(["\']{group_name}["\']\s*:\s*["\']?){group}'.format(group_name=group_name, group=user_regex),
+ 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format(
+ group='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)),
+ 'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep),
+ 'user2': r'/(home|user)/{user}'.format(user=user_regex),
'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex),
- 'user4': r'(?/) exclude patterns '>/' where we have already matched and token replaced
'user6': r'(?/){user}@'.format(user=user_regex),
- 'password': r'(-?{pass_word_phrase}{sep}){pw}'\
+ 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'\
+ .format(user_name=user_name, user=user_regex),
+ #'user8': r'arn:aws:iam::\d{12}:user/{user}'.format(user=user_regex),
+ 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format(
+ user='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)),
+ 'password': r'([\.-]?{pass_word_phrase}{sep}){pw}'\
.format(pass_word_phrase=pass_word_phrase,
sep=arg_sep,
pw=password_quoted),
@@ -308,8 +383,16 @@ def __init__(self):
sep=arg_sep,
pass_word_phrase=pass_word_phrase,
pw=password_quoted),
- 'ip3': aws_host_ip_regex + r'(?!-\d)',
- 'ip_prefix': r'{}(?!\.\d+\.\d+)'.format(ip_prefix_regex),
+ 'password4': r'([\.-]?(?:api-?)?token{sep}){pw}'\
+ .format(sep=arg_sep,
+ pw=password_quoted),
+ 'ip': r'(?).*?$',
'cisco3': r'\ssecret\s.*?$',
'cisco4': r'\smd5\s+.*?$',
'cisco5': r'\scommunity\s+.*$',
@@ -366,30 +449,61 @@ def __init__(self):
ldap_lambda_lowercase = lambda m: r'{}<{}>'.format(m.group(1), m.group(2).lower())
# will auto-infer replacements to not have to be explicit, use this only for override mappings
self.replacements = {
+ # arn:partition:service:region:account-id:resource-id
+ # arn:partition:service:region:account-id:resource-type/resource-id
+ # arn:partition:service:region:account-id:resource-type:resource-id
+ 'aws': r'\1\2<\3>',
+ 'aws2': r'\1:',
+ 'aws3': r'',
+ 'aws4': r'',
+ 'aws5': r'',
+ 'aws6': r'',
+ 'aws7': r'',
+ 'aws8': r'\1:///',
+ 'aws9': r'',
+ #'aws10': r'ec2-x-x-x-x.\1',
+ 'db': r'\1',
+ 'db2': r'\1',
+ 'db3': r'\1',
+ 'db4': r'\1<\2>',
+ 'db5': r'/user/hive/warehouse/.db/',
+ 'generic': r'\1://',
+ 'generic2': r'\1',
+ 'generic3': r'\1',
+ 'generic4': r'\1',
+ 'generic5': r'\1',
'hostname': r':\2',
#'hostname2': '',
'hostname2': r'',
'hostname3': r'\1',
'hostname4': r'\\\\',
'hostname5': r'\1',
+ 'hostname6': r'\1',
'domain2': '@',
'port': ':',
'user': r'\1',
- 'user2': '/home/',
+ 'user2': r'/\1/',
'user3': r'\1',
'user4': r'\\',
'user5': 'for user ',
'user6': '@',
+ 'user7': r'\1',
+ 'user8': r'\1\2',
'group': r'\1',
'group2': r'\1',
'group3': r'for group ',
+ 'group4': r'\1',
+ 'group5': r'\1\2',
'password': r'\1',
'password2': r'\1:',
'password3': r'\1\2',
+ 'password4': r'\1',
'ip': r'/',
'ip2': r'',
'ip3': r'',
- 'ip_prefix': r'.',
+ 'ip_prefix': r'.\1/',
+ 'ip_prefix2': r'.\1',
+ 'ip_prefix3': r'.\1',
'subnet_mask': r'',
'kerberos': r'host/\1@',
'kerberos2': r'host/@',
@@ -402,12 +516,12 @@ def __init__(self):
'ldap2': ldap_lambda_lowercase,
'ldap3': ldap_lambda_lowercase,
'proxy': r'proxy port ',
- 'proxy2': r'Trying ',
- 'proxy3': r'Connected to () port ',
+ 'proxy2': r'Connected to () port ',
+ 'proxy3': r'\1',
'http_auth': r'$1:@',
'http_auth2': r'\1\'\2\3/',
'http_auth3': r'Authorization: Basic ',
- 'proxy4': r'\1',
+ 'http_auth4': r'\1',
'cisco': r'username password ',
'cisco2': r'password ',
'cisco3': r'secret ',
@@ -437,6 +551,12 @@ def add_options(self):
self.add_opt('-a', '--all', action='store_true',
help='Apply all anonymizations (careful this includes --host which can be overzealous and ' + \
'match too many things, in which case try more targeted anonymizations below)')
+ self.add_opt('-w', '--aws', action='store_true',
+ help='Apply AWS anonymizations (access/secret keys, STS tokens, ARNs, buckets, security groups)')
+ self.add_opt('-b', '--db', '--database', action='store_true',
+ help='Apply database anonymizations (db name, instance name)')
+ self.add_opt('-g', '--generic', action='store_true',
+ help='Apply generic anonymizations (file://, key, cluster name / id etc)')
self.add_opt('-C', '--custom', action='store_true',
help='Apply custom phrase anonymization (add your Name, Company Name etc to the list of ' + \
'blacklisted words/phrases one per line in anonymize_custom.conf). Matching is case ' + \
@@ -465,7 +585,7 @@ def add_options(self):
'debugging, these are salted and truncated to be indistinguishable from temporal docker ' + \
'container IDs, but someone with enough computing power and time could theoretically ' + \
'calculate the source hostnames so don\'t put these on the public internet, it is more ' + \
- 'for private vendor tickets)')
+ 'for private vendor tickets')
self.add_opt('-d', '--domain', action='store_true',
help='Apply domain format anonymization')
self.add_opt('-F', '--fqdn', action='store_true',
@@ -484,26 +604,22 @@ def add_options(self):
r'http://username:password\@ => ' + \
r'http://:\@. Also works with https://')
self.add_opt('-K', '--kerberos', action='store_true',
- help=r'Kerberos 5 principals in the form @ or /@ ' + \
+ help=r'Apply Kerberos anonymizations eg. @, /@ ' + \
'(where must match a valid domain name - otherwise use --custom and populate ' + \
- r'anonymize_custom.conf). These kerberos principals are anonymizebed to ' + \
- '. There is a special exemption for Hadoop Kerberos principals such ' + \
- 'as NN/_HOST@ which preserves the literal \'_HOST\' instance since that\'s ' + \
- 'useful to know for debugging, the principal and realm will still be anonymizebed in ' + \
- 'those cases (if wanting to retain NN/_HOST then use --domain instead of --kerberos). ' + \
- 'This is applied before --email in order to not prevent the email replacement leaving ' + \
- r'this as user/host\@realm to user/, which would have exposed \'user\'' + \
- '. Auto enables --email, --domain and --fqdn')
+ r'anonymize_custom.conf). Hadoop principals preserve the generic _HOST placeholder eg. ' + \
+ '/_HOST@ (if wanting to retain full prefix eg. NN/_HOST then use ' + \
+ '--domain instead of --kerberos). --kerberos auto-enables --email, --domain and --fqdn')
self.add_opt('-L', '--ldap', action='store_true',
- help='Apply LDAP anonymization (~100 attribs eg. CN, DN, OU, UID, sAMAccountName, memberOf...)')
+ help='Apply LDAP anonymization ' + \
+ '(~100 attribs eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...)')
self.add_opt('-E', '--email', action='store_true',
help='Apply email format anonymization')
- self.add_opt('-x', '--proxy', action='store_true',
- help='Apply anonymization to remove proxy host, user etc (eg. from curl -iv output). You ' + \
- 'should probably also apply --ip and --host if using this. Auto enables --http-auth')
+ #self.add_opt('-x', '--proxy', action='store_true',
+ # help='Apply anonymization to remove proxy host, user etc (eg. from curl -iv output). You ' + \
+ # 'should probably also apply --ip and --host if using this. Auto enables --http-auth')
self.add_opt('-N', '--network', action='store_true',
help='Apply all network anonymization, whether Cisco, ScreenOS, JunOS for secrets, auth, ' + \
- 'usernames, passwords, md5s, PSKs, AS, SNMP etc.')
+ 'usernames, passwords, md5s, PSKs, AS, SNMP community strings etc.')
self.add_opt('-c', '--cisco', action='store_true',
help='Apply Cisco IOS/IOS-XR/NX-OS configuration format anonymization')
self.add_opt('-s', '--screenos', action='store_true',
@@ -539,11 +655,19 @@ def process_options(self):
if _ == 'ip_prefix':
continue
self.anonymizations[_] = True
+ if self.get_opt('ip_prefix'):
+ self.anonymizations['ip_prefix'] = True
+ self.anonymizations['ip'] = False
else:
for _ in self.anonymizations:
+ # pylint: disable=no-else-continue
if _ in ('subnet_mask', 'mac', 'group'):
continue
- self.anonymizations[_] = self.get_opt(_)
+ elif _ == 'database':
+ self.anonymizations['db'] = True
+ else:
+ self.anonymizations[_] = self.get_opt(_)
+ log.debug('anonymization enabled %s = %s', _, bool(self.anonymizations[_]))
self._process_options_host()
self._process_options_network()
self._process_options_exceptions()
@@ -593,8 +717,8 @@ def _process_options_host(self):
self.anonymizations['email'] = True
self.anonymizations['domain'] = True
self.anonymizations['fqdn'] = True
- if self.anonymizations['proxy']:
- self.anonymizations['http_auth'] = True
+ #if self.anonymizations['proxy']:
+ # self.anonymizations['http_auth'] = True
def _process_options_network(self):
if self.anonymizations['network']:
@@ -651,7 +775,7 @@ def load_file(filename, boundary=False):
log.warning('ignoring invalid regex from %s: %s', os.path.basename(filename), line)
continue
if boundary:
- line = r'(\b|[^A-Za-z])' + line + r'(\b|[^A-Za-z])'
+ line = r'(?:(?<=\b)|(?<=[^A-Za-z]))' + line + r'(?=\b|[^A-Za-z])'
regex_list.append(line)
raw = '|'.join(regex_list)
#log.debug('custom_raw: %s', raw)
@@ -668,45 +792,49 @@ def run(self):
# allow to easily switch pre-compilation on/off for testing
# testing shows on a moderate sized file that it is a couple secs quicker to use pre-compiled regex
def compile(self, name, regex):
+ log.debug(f"compiling regex '{name}' = '{regex}'")
self.regex[name] = re.compile(regex, re.I)
#self.regex[name] = regex
def prepare_regex(self):
self.compile('hostname',
- r'(? 2018-01-:00:00
- r'(?!\d+T\d+:\d+)' + \
- r'(?!\d+[^A-Za-z0-9]|' + \
- self.custom_ignores_raw + ')' + \
- '(' + hostname_regex + ')' + \
+ r'(?!\d+T\d+:\d+)' +
+ r'(?!\d+[^A-Za-z0-9]|' +
+ self.custom_ignores_raw + ')' +
+ '(' + hostname_regex + ')' +
self.negative_host_lookbehind + r':(\d{1,5}(?!\.?\w))',
)
self.compile('domain',
- r'(?!' + self.custom_ignores_raw + ')' + \
- domain_regex_strict + \
- r'(?!\.[A-Za-z])(\b|$)' + \
+ # don't match java -some.net.property
+ #r'(?\2', line)
+ i += 1
+ line = regex.sub(r'', line)
#line = re.sub(regex, r'\1\2', line)
+ log.debug('anonymize_custom: %s => %s', i, line)
return line
@staticmethod
diff --git a/anonymize_custom.conf b/anonymize_custom.conf
index 28315e609..b50186df7 100644
--- a/anonymize_custom.conf
+++ b/anonymize_custom.conf
@@ -2,7 +2,7 @@
# Author: Hari Sekhon
# Date: 2013-07-18 22:00:56 +0100 (Thu, 18 Jul 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -22,7 +22,7 @@
###################################
# Personal Names
#
-# add your name here if you don't want your name appearing in generic free text (--cisco/--screenos/--junos will already strip out usernames from Cisco/Juniper configs anyway based on the syntax of those configuration languages, if they require an update you should instead raise a ticket for a more generic fix to https://github.com/harisekhon/devops-perl-tools/issues, which is better than listing every possible user by hand).
+# add your name here if you don't want your name appearing in generic free text (--cisco/--screenos/--junos will already strip out usernames from Cisco/Juniper configs anyway based on the syntax of those configuration languages, if they require an update you should instead raise a ticket for a more generic fix to https://github.com/HariSekhon/DevOps-Perl-tools/issues, which is better than listing every possible user by hand).
# This is more for a short problem log where your username is in free text and can't be auto-determined by context.
hari.?sekhon
hari
diff --git a/anonymize_ignore.conf b/anonymize_ignore.conf
index 544a1f777..d6c4aec78 100644
--- a/anonymize_ignore.conf
+++ b/anonymize_ignore.conf
@@ -2,7 +2,7 @@
# Author: Hari Sekhon
# Date: 2013-07-18 22:00:56 +0100 (Thu, 18 Jul 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
diff --git a/anonymize_parallel.sh b/anonymize_parallel.sh
new file mode 100755
index 000000000..abb7e9d21
--- /dev/null
+++ b/anonymize_parallel.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-03-05 18:27:00 +0000 (Tue, 05 Mar 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# shellcheck disable=SC1090
+. "$srcdir/bash-tools/lib/utils.sh"
+
+# re-establish srcdir local to this script since util.sh include brings its own srcdir
+srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# shellcheck disable=SC2120
+usage(){
+ if [ -n "$*" ]; then
+ echo "$@" >&2
+ echo >&2
+ fi
+ cat >&2 <
+
+-p --parallelism Number of parts to split files in to and anonymize in parallel before reconstituting
+-h --help Show usage and exit
+EOF
+ exit 3
+}
+
+parallelism="${PARALLELISM:-$(cpu_count)}"
+
+file_list=""
+
+while [ $# -gt 0 ]; do
+ # shellcheck disable=SC2119
+ case $1 in
+ -p|--parallel) parallelism="$2"
+ shift
+ ;;
+ -h|--help|-*) usage
+ ;;
+ *) file_list="$file_list $1"
+ ;;
+ esac
+ shift
+done
+
+for filename in $file_list; do
+ echo
+ echo "Processing file '$filename':"
+ echo
+ echo "Removing any pre-existing parts:"
+ rm -v "$filename".* 2>/dev/null || :
+ echo
+ "$srcdir/bash-tools/bin/split.sh" --parts "$parallelism" "$filename"
+ echo "Anonymizing parts"
+ for file_part in "$filename".*; do
+ cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized"
+ echo "$cmd"
+ done |
+ parallel -j "$parallelism"
+ echo "Concatenating parts"
+ cat "$filename".*.anonymized > "$filename".anonymized
+ echo
+ echo "Removing parts:"
+ rm -v "$filename".*.anonymized || :
+ rm -v "$filename".[a-z0-9][a-z0-9] "$filename".[a-z0-9][a-z0-9][a-z0-9] 2>/dev/null || :
+ echo
+ echo "Anonymized file ready: $filename.anonymized"
+ echo
+done
diff --git a/aws_s3_presign.py b/aws_s3_presign.py
new file mode 100755
index 000000000..e1addfbbc
--- /dev/null
+++ b/aws_s3_presign.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-01-14 17:45:38 +0000 (Tue, 14 Jan 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Generate and print an S3 pre-signed URL
+
+Can do the same with the following AWS CLI command:
+
+aws s3 presign s3:/// [--expires-in ]
+
+Will generate a pre-signed URL even when the bucket and object key don't exist!
+
+(you will get a runtime error when requesting the link that the bucket or object doesn't exist)
+
+
+Uses the Boto library, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import sys
+import boto3
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.1'
+
+def main():
+ parser = argparse.ArgumentParser(
+ description='Generate an AWS S3 pre-signed URL to access an S3 object without logging in')
+ parser.add_argument('bucket', help='Bucket Name')
+ parser.add_argument('key', help='Key')
+ parser.add_argument('expiration', nargs='?', default=3600, help='Expiration of URL in seconds')
+ args = parser.parse_args()
+
+ # more useful if doing this programmatically as we can do this on the command line via AWS CLI
+ conn = boto3.client('s3')
+ url = conn.generate_presigned_url(
+ 'get_object',
+ Params={
+ 'Bucket': args.bucket,
+ 'Key': args.key
+ },
+ ExpiresIn=args.expiration
+ )
+ print(url)
+
+
+if __name__ == '__main__':
+ try:
+ main()
+ except KeyboardInterrupt:
+ print('Control-C...', file=sys.stderr)
diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py
new file mode 100755
index 000000000..be218c2db
--- /dev/null
+++ b/aws_users_access_key_age.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-13 17:24:40 +0000 (Fri, 13 Dec 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Lists all AWS IAM users keys along with their ages, optionally filtering any older than a given number of days
+
+Output format is:
+
+
+
+Status is Active or Inactive
+
+Validated compared to xls report download from Trusted Advisor -> Security -> IAM Access Key Rotation
+
+Uses the Boto library, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies
+
+See also:
+
+ aws_users_access_key_age.sh - simpler version in the adjacent DevOps Bash Tools repo without age filtering
+ - https://github.com/HariSekhon/DevOps-Bash-tools
+
+Advanced Nagios Plugins (https://github.com/HariSekhon/Nagios-Plugins)
+
+ check_aws_access_keys_age.py
+ check_aws_access_keys_disabled.py
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import datetime
+import os
+import sys
+from math import ceil
+import boto3
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_float
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3.0'
+
+class AWSUsersAccessKeysAge(CLI):
+
+ def __init__(self):
+ super(AWSUsersAccessKeysAge, self).__init__()
+ self.age = None
+ self.now = None
+ self.only_active_keys = False
+ self.timeout_default = 300
+
+ def add_options(self):
+ self.add_opt('-a', '--age', help='Return keys older than N days')
+ self.add_opt('-o', '--only-active', action='store_true', help='Return only keys with Active status')
+
+ def process_args(self):
+ self.only_active_keys = self.get_opt('only_active')
+ self.age = self.get_opt('age')
+ if self.age:
+ validate_float(self.age, 'age')
+ self.age = float(self.age)
+
+ def run(self):
+ iam = boto3.client('iam')
+ user_paginator = iam.get_paginator('list_users')
+ self.now = datetime.datetime.utcnow()
+ for users_response in user_paginator.paginate():
+ for user_item in users_response['Users']:
+ username = user_item['UserName']
+ key_paginator = iam.get_paginator('list_access_keys')
+ for keys_response in key_paginator.paginate(UserName=username):
+ self.process_key(keys_response, username)
+ log.info('Completed')
+
+ def process_key(self, keys_response, username):
+ #assert not keys_response['IsTruncated']
+ for access_key_item in keys_response['AccessKeyMetadata']:
+ assert username == access_key_item['UserName']
+ status = access_key_item['Status']
+ if self.only_active_keys and status != 'Active':
+ continue
+ create_date = access_key_item['CreateDate']
+ # already cast to datetime.datetime with tzinfo
+ #create_datetime = datetime.datetime.strptime(create_date, '%Y-%m-%dT%H:%M:%S%z')
+ # removing tzinfo for comparison to avoid below error
+ # - both areOA UTC and this doesn't make much difference anyway
+ # TypeError: can't subtract offset-naive and offset-aware datetimes
+ age_timedelta = self.now - create_date.replace(tzinfo=None)
+ age_days = int(ceil(age_timedelta.total_seconds() / 86400.0))
+ if self.age:
+ if age_days < self.age:
+ continue
+ print('{user:20}\t{status:8}\t{date}\t({days:>3} days)'.format(
+ user=username,
+ status=status,
+ date=create_date,
+ days=age_days))
+
+
+if __name__ == '__main__':
+ AWSUsersAccessKeysAge().main()
diff --git a/aws_users_last_used.py b/aws_users_last_used.py
new file mode 100755
index 000000000..e06a7bcdb
--- /dev/null
+++ b/aws_users_last_used.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019)
+#
+# https://github.com/HariSekhon/Nagios-Plugins
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Find AWS IAM user accounts last used age in days using the most recently used among their timestamp and access keys
+
+Optionally filters to only users > N days old to find old user accounts
+
+Generates an IAM credential report, then parses it to determine the time since each user's password
+and access keys were last used
+
+Requires iam:GenerateCredentialReport on resource: *
+
+Output:
+
+
+
+Uses the Boto python library, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools
+
+- https://github.com/HariSekhon/DevOps-Bash-tools
+- https://github.com/HariSekhon/Nagios-Plugins
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import csv
+import os
+import sys
+import time
+import traceback
+from datetime import datetime
+from io import StringIO
+from math import floor
+import boto3
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_int
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.1'
+
+
+class AWSUsersLastUsed(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(AWSUsersLastUsed, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.age = None
+ self.now = None
+ self.timeout_default = 300
+ self.msg = 'AWSUsersLastUsed msg not defined'
+
+ def add_options(self):
+ self.add_opt('-a', '--age', type=float,
+ help='Filters to show only accounts last used more than N days ago')
+
+ def process_args(self):
+ self.no_args()
+ self.age = self.get_opt('age')
+ if self.age is not None:
+ validate_int(self.age, 'age')
+
+ def run(self):
+ iam = boto3.client('iam')
+ log.info('generating credentials report')
+ while True:
+ result = iam.generate_credential_report()
+ log.debug('%s', result)
+ if result['State'] == 'COMPLETE':
+ log.info('credentials report generated')
+ break
+ log.info('waiting for credentials report')
+ time.sleep(1)
+ #try:
+ result = iam.get_credential_report()
+ #except ClientError as _:
+ # raise
+ csv_content = result['Content']
+ log.debug('%s', csv_content)
+ filehandle = StringIO(unicode(csv_content))
+ filehandle.seek(0)
+ csvreader = csv.reader(filehandle)
+ headers = next(csvreader)
+ assert headers[0] == 'user'
+ assert headers[4] == 'password_last_used'
+ assert headers[10] == 'access_key_1_last_used_date'
+ assert headers[15] == 'access_key_2_last_used_date'
+ self.now = datetime.utcnow()
+ for row in csvreader:
+ self.process_user(row)
+
+ def process_user(self, row):
+ log.debug('processing user: %s', row)
+ user = row[0]
+ password_last_used = row[4]
+ access_key_1_last_used_date = row[10]
+ access_key_2_last_used_date = row[15]
+ log.debug('user: %s, password_last_used: %s, access_key_1_last_used_date: %s, access_key_2_last_used_date: %s',
+ user, password_last_used, access_key_1_last_used_date, access_key_2_last_used_date)
+ min_age = None
+ for _ in [password_last_used, access_key_1_last_used_date, access_key_2_last_used_date]:
+ if _ == 'N/A':
+ continue
+ # %z not working in Python 2.7 but we already know it's +00:00
+ _datetime = datetime.strptime(_.split('+')[0], '%Y-%m-%dT%H:%M:%S')
+ age_timedelta = self.now - _datetime.replace(tzinfo=None)
+ age_days = int(floor(age_timedelta.total_seconds() / 86400.0))
+ if min_age is None or age_days < min_age:
+ min_age = age_days
+ if self.age and min_age <= self.age:
+ return
+ print('{user:20}\t{days:>3}\t{password_last_used:25}\t'\
+ .format(user=user,
+ days=min_age,
+ password_last_used=password_last_used) +
+ '{access_key_1_last_used_date:25}\t{access_key_2_last_used_date:25}'\
+ .format(access_key_1_last_used_date=access_key_1_last_used_date,
+ access_key_2_last_used_date=access_key_2_last_used_date))
+
+
+if __name__ == '__main__':
+ AWSUsersLastUsed().main()
diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py
new file mode 100755
index 000000000..6810abd17
--- /dev/null
+++ b/aws_users_pw_last_used.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-19 11:43:25 +0000 (Thu, 19 Dec 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Lists all AWS IAM users dates since their passwords were last used, optionally filtering
+for users whose passwords haven't been used in > N days
+
+Output format is:
+
+
+
+Uses Boto, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+This version adds date parsing for finding keys older than a given time for enforcing periodic key rotation policies
+
+See also the DevOps Bash Tools repo and The Advanced Nagios Plugins Collection for similar tools
+
+https://github.com/HariSekhon/DevOps-Bash-tools
+
+https://github.com/HariSekhon/Nagios-Plugins
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import datetime
+import os
+import sys
+from math import floor
+import boto3
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_float
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+class AWSUsersPasswordLastUsed(CLI):
+
+ def __init__(self):
+ super(AWSUsersPasswordLastUsed, self).__init__()
+ self.age = None
+ self.now = None
+ self.timeout_default = 300
+
+ def add_options(self):
+ self.add_opt('-a', '--age', help='Return users with passwords last used more than N days ago')
+
+ def process_args(self):
+ self.age = self.get_opt('age')
+ if self.age:
+ validate_float(self.age, 'age')
+ self.age = float(self.age)
+
+ def run(self):
+ iam = boto3.client('iam')
+ user_paginator = iam.get_paginator('list_users')
+ self.now = datetime.datetime.utcnow()
+ for users_response in user_paginator.paginate():
+ for user_item in users_response['Users']:
+ log.debug(user_item)
+ self.process_password_last_used(user_item)
+ log.info('Completed')
+
+ def process_password_last_used(self, user_item):
+ # already cast to datetime.datetime with tzinfo
+ user = user_item['UserName']
+ if 'PasswordLastUsed' in user_item:
+ password_last_used = user_item['PasswordLastUsed']
+ # removing tzinfo for comparison to avoid below error
+ # - both are UTC and this doesn't make much difference anyway
+ # TypeError: can't subtract offset-naive and offset-aware datetimes
+ datetime_delta = self.now - password_last_used.replace(tzinfo=None)
+ days = int(floor(datetime_delta.total_seconds() / 86400))
+ if self.age and days <= self.age:
+ return
+ else:
+ password_last_used = 'N/A'
+ days = 'N/A'
+ print('{user:20s}\t{password_last_used:25s}\t({days:>3} days)'.format(
+ user=user,
+ password_last_used=str(password_last_used), # without str() format string breaks with :25
+ days=days))
+
+
+if __name__ == '__main__':
+ AWSUsersPasswordLastUsed().main()
diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py
new file mode 100755
index 000000000..befb152ee
--- /dev/null
+++ b/aws_users_unused_access_keys.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019)
+#
+# https://github.com/HariSekhon/Nagios-Plugins
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Find AWS IAM user access keys that haven't been used in N days and keys that have never been used
+
+Generates an IAM credential report, then parses it to determine the time since each user's password
+and access keys were last used
+
+Requires iam:GenerateCredentialReport on resource: *
+
+Output:
+
+
+
+Uses the Boto python library, read here for the list of ways to configure your AWS credentials:
+
+ https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
+
+See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools
+
+- https://github.com/HariSekhon/DevOps-Bash-tools
+- https://github.com/HariSekhon/Nagios-Plugins
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import csv
+import os
+import sys
+import time
+import traceback
+from datetime import datetime
+from io import StringIO
+from math import floor
+import boto3
+from botocore.exceptions import ClientError
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_int
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.2.1'
+
+
+class AWSUnusedAccessKeys(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(AWSUnusedAccessKeys, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.age = None
+ self.now = None
+ self.only_active = False
+ self.timeout_default = 300
+ self.msg = 'AWSUnusedAccessKeys msg not defined'
+
+ def add_options(self):
+ self.add_opt('-a', '--age', type=float, default=30,
+ help='Show only access keys not used in the last N days (default: 30)')
+ self.add_opt('-o', '--only-active', action='store_true',
+ help='Only show access keys that are active')
+
+ def process_args(self):
+ self.no_args()
+ self.age = self.get_opt('age')
+ if self.age is not None:
+ validate_int(self.age, 'age')
+
+ def run(self):
+ iam = boto3.client('iam')
+ log.info('generating credentials report')
+ while True:
+ result = iam.generate_credential_report()
+ log.debug('%s', result)
+ if result['State'] == 'COMPLETE':
+ log.info('credentials report generated')
+ break
+ log.info('waiting for credentials report')
+ time.sleep(1)
+ try:
+ result = iam.get_credential_report()
+ except ClientError as _:
+ raise
+ csv_content = result['Content']
+ log.debug('%s', csv_content)
+ filehandle = StringIO(unicode(csv_content))
+ filehandle.seek(0)
+ csvreader = csv.reader(filehandle)
+ headers = next(csvreader)
+ log.debug('headers: %s', headers)
+ assert headers[0] == 'user'
+ assert headers[8] == 'access_key_1_active'
+ assert headers[9] == 'access_key_1_last_rotated'
+ assert headers[10] == 'access_key_1_last_used_date'
+ assert headers[13] == 'access_key_2_active'
+ assert headers[14] == 'access_key_2_last_rotated'
+ assert headers[15] == 'access_key_2_last_used_date'
+ self.now = datetime.utcnow()
+ for row in csvreader:
+ self.process_user(row)
+
+ def process_user(self, row):
+ log.debug('processing user: %s', row)
+ user = row[0]
+ access_keys = {1:{}, 2:{}}
+ access_keys[1]['active'] = row[8]
+ access_keys[1]['last_used_date'] = row[10]
+ access_keys[1]['last_rotated'] = row[9]
+ access_keys[2]['active'] = row[13]
+ access_keys[2]['last_rotated'] = row[14]
+ access_keys[2]['last_used_date'] = row[15]
+ for key in [1, 2]:
+ active = access_keys[key]['active']
+ if not isinstance(active, bool):
+ assert active in ('true', 'false')
+ active = active.lower() == 'true'
+ created = access_keys[key]['last_rotated']
+ last_used = access_keys[key]['last_used_date']
+ log.debug('user: %s, key: %s, active: %s, created: %s, last_used_date: %s, ',
+ user,
+ key,
+ active,
+ created,
+ last_used
+ )
+ if not active and self.only_active:
+ continue
+ if last_used == 'N/A':
+ if created == 'N/A':
+ continue
+ self.print_key(user, key, active, 'N/A', last_used, created)
+ continue
+ # %z not working in Python 2.7 but we already know it's +00:00
+ _datetime = datetime.strptime(last_used.split('+')[0], '%Y-%m-%dT%H:%M:%S')
+ age_timedelta = self.now - _datetime.replace(tzinfo=None)
+ age_days = int(floor(age_timedelta.total_seconds() / 86400.0))
+ if age_days > self.age:
+ self.print_key(user, key, active, age_days, last_used, created)
+
+ # pylint: disable=too-many-arguments
+ def print_key(self, user, key, active, age_days, last_used, created):
+ log.debug('access key %s, active: %s, last used %s days ago > %s', key, active, age_days, self.age)
+ print('{user:20}\t{key}\t{active}\t{days:>3}\t{last_used:25}\t{created}'\
+ .format(user=user,
+ key=key,
+ active='Active' if active else 'Inactive',
+ days=age_days,
+ last_used=last_used,
+ created=created
+ )
+ )
+
+
+if __name__ == '__main__':
+ AWSUnusedAccessKeys().main()
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
new file mode 100644
index 000000000..b430c58dd
--- /dev/null
+++ b/azure-pipelines.yml
@@ -0,0 +1,70 @@
+# vim:ts=2:sts=2:sw=2:et
+#
+# Author: Hari Sekhon
+# Date: Sun Feb 23 19:02:10 2020 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# A z u r e D e v O p s P i p e l i n e
+# ============================================================================ #
+
+# https://aka.ms/yaml
+
+trigger:
+ - master
+
+variables:
+ # ubuntu version
+ os_version: '22.04'
+
+pool:
+ # there is no /dev/stderr on this azure build!
+ #vmImage: 'ubuntu-latest'
+ #vmImage: 'ubuntu-22.04'
+ vmImage: 'ubuntu-$(os_version)'
+
+# unprivileged container without sudo, cannot install dependencies
+#container: ubuntu:22.04
+
+steps:
+ - script: cat /etc/*-release
+ displayName: OS Release
+
+ # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName'
+ - script: env | sort
+ displayName: Environment
+
+ # doesn't work in container due to unprivileged execution and lack of sudo
+ #- script: sudo apt-get update && sudo apt-get install -y git make
+ # displayName: install git & make
+
+ #- script: make
+ # displayName: build
+
+ # doesn't work in vmImage build due to lack of access to normal /dev/stderr device
+ # tee: /dev/stderr: No such device or address
+ #- script: make test
+ # displayName: test
+
+ # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-(
+ - script: |
+ sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c '
+ set -ex
+ cd /code
+ setup/ci_bootstrap.sh
+ if [ -x setup/ci_git_set_dir_safe.sh ]; then
+ setup/ci_git_set_dir_safe.sh
+ fi
+ make init
+ make ci test
+ '
+ displayName: docker build
diff --git a/bash-tools b/bash-tools
index a2300e7cc..11dee29ce 160000
--- a/bash-tools
+++ b/bash-tools
@@ -1 +1 @@
-Subproject commit a2300e7cc4d4fdcd5f87ddba5da99d750ba9d3c5
+Subproject commit 11dee29cea607445270d8bd675d1bcdecb069c14
diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
new file mode 100644
index 000000000..cdf5243ee
--- /dev/null
+++ b/bitbucket-pipelines.yml
@@ -0,0 +1,38 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-24 17:08:57 +0000 (Mon, 24 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# B i t b u c k e t C I / C D P i p e l i n e
+# ============================================================================ #
+
+# Reference:
+#
+# https://support.atlassian.com/bitbucket-cloud/docs/configure-bitbucket-pipelinesyml/
+
+# Languages:
+#
+# https://confluence.atlassian.com/x/5Q4SMw
+
+# You can specify a custom docker image from Docker Hub as your build environment.
+image: atlassian/default-image:2
+
+pipelines:
+ default:
+ - step:
+ script:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
diff --git a/boot b/boot
new file mode 120000
index 000000000..4092e5539
--- /dev/null
+++ b/boot
@@ -0,0 +1 @@
+setup/bootstrap.sh
\ No newline at end of file
diff --git a/buddy.yml b/buddy.yml
new file mode 100644
index 000000000..e0b57451f
--- /dev/null
+++ b/buddy.yml
@@ -0,0 +1,48 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# B u d d y C I
+# ============================================================================ #
+
+# https://buddy.works/docs/yaml/yaml-schema
+
+---
+- pipeline: "Build"
+ trigger_mode: "ON_EVERY_PUSH"
+ ref_name: "master"
+ ref_type: "BRANCH"
+ target_site_url: "https://github.com/HariSekhon/DevOps-Python-tools"
+ trigger_condition: "ALWAYS"
+ actions:
+ - action: "Execute: make ci test"
+ type: "BUILD"
+ working_directory: "/buddy/DevOps-Python-tools"
+ docker_image_name: "library/ubuntu"
+ docker_image_tag: "18.04"
+ #setup_commands:
+ # this step gets cached, which results in
+ # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?
+ # - apt update
+ # - apt install -qy git make
+ execute_commands:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
+ volume_mappings:
+ - "/:/buddy/DevOps-Python-tools"
+ shell: "BASH"
+ trigger_condition: "ALWAYS"
diff --git a/center.py b/center.py
index 752db7f6a..379353e68 100755
--- a/center.py
+++ b/center.py
@@ -1,17 +1,19 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
+# args: -s <<< "Auth & Config"
+# args: -s <<< "GKE Clusters"
#
# Author: Hari Sekhon
# Date: 2016-01-29 21:05:38 +0000 (Fri, 29 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -42,7 +44,8 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.4.0'
+__version__ = '0.5.1'
+
class Center(CLI):
@@ -51,8 +54,13 @@ def __init__(self):
super(Center, self).__init__()
# Python 3.x
# super().__init__()
- self.re_bound = re.compile(r'(\b)')
+ # this doesn't put enough spaces around ampersands, eg. in "Auth & Config"
+ #self.re_bound = re.compile(r'(\b)')
+ self.re_spaces = re.compile(r'(\s)')
+ self.re_multiple_spaces = re.compile(r'(\s){2}')
self.re_chars = re.compile(r'([^\s])(?!\s)')
+ self.re_chars_spaced = re.compile(r'([^\s])\s')
+ self.timeout_default = None
def add_options(self):
self.add_opt('-w', '--width', default=80, type='int', metavar='',
@@ -61,11 +69,16 @@ def add_options(self):
help='No comment prefix handling')
self.add_opt('-s', '--space', action='store_true', default=False,
help='Space all chars out, makes bigger headings')
+ self.add_opt('-u', '--unspace', action='store_true', default=False,
+ help='Removes spaces betweeen chars out, the inverse of --space')
def run(self):
log_option('width', self.get_opt('width'))
log_option('no comment prefix', self.get_opt('no_comment'))
log_option('space chars', self.get_opt('space'))
+ log_option('unspace chars', self.get_opt('unspace'))
+ if self.get_opt('space') and self.get_opt('unspace'):
+ self.usage("--space and --unspace are mutually exclusive!")
if self.args:
self.process_line(' '.join(self.args))
else:
@@ -73,10 +86,16 @@ def run(self):
self.process_line(line)
def space(self, line):
- line = self.re_bound.sub(r' ', line)
+ #line = self.re_bound.sub(r' ', line)
+ line = self.re_spaces.sub(r'\1\1\1', line)
line = self.re_chars.sub(r'\1 ', line)
return line
+ def unspace(self, line):
+ line = self.re_chars_spaced.sub(r'\1', line)
+ line = self.re_multiple_spaces.sub(r'\1', line)
+ return line
+
def process_line(self, line):
char = ''
if not line:
@@ -85,7 +104,7 @@ def process_line(self, line):
char = ' '
# preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it
line = line.strip()
- if isChars(line[0], '#'):
+ if line and isChars(line[0], '#'):
char = line[0]
line = line.lstrip(char)
elif len(line) > 1 and isChars(line[0:1], '/'):
@@ -96,9 +115,12 @@ def process_line(self, line):
line = line.lstrip(char)
if self.get_opt('space'):
line = self.space(line)
+ if self.get_opt('unspace'):
+ line = self.unspace(line)
line = line.strip()
side = int(max((self.get_opt('width') - len(line)) / 2, 0))
print(char + ' ' * side + line)
+
if __name__ == '__main__':
Center().main()
diff --git a/cicd/.concourse.yml b/cicd/.concourse.yml
new file mode 100644
index 000000000..92847121e
--- /dev/null
+++ b/cicd/.concourse.yml
@@ -0,0 +1,65 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-03-21 11:06:48 +0000 (Sat, 21 Mar 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C o n c o u r s e C I
+# ============================================================================ #
+
+# https://concourse-ci.org/golang-library-example.html
+
+# https://resource-types.concourse-ci.org/
+# https://concourse-ci.org/resource-types.html
+resources:
+ - name: github
+ icon: github-circle
+ type: git
+ source:
+ uri: https://github.com/HariSekhon/DevOps-Python-tools
+ branch: master
+ #- name: daily
+ # type: time
+ # source:
+ # interval: 1d
+
+# https://concourse-ci.org/jobs.html
+jobs:
+ - name: build
+ public: false
+ plan:
+ - get: github # from resource above
+ trigger: true
+ #version: every # build every git commit, default: latest
+ - task: build
+ config:
+ platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ repository: ubuntu
+ tag: latest
+ inputs:
+ - name: github
+ path: code
+ params:
+ CONCOURSE: 1
+ run:
+ path: /bin/bash
+ args:
+ - -c
+ - |
+ cd code &&
+ setup/ci_bootstrap.sh &&
+ make init &&
+ make ci test
diff --git a/cicd/.gocd.yml b/cicd/.gocd.yml
new file mode 100644
index 000000000..65583a818
--- /dev/null
+++ b/cicd/.gocd.yml
@@ -0,0 +1,94 @@
+# vim:ts=2:sts=2:sw=2:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G o C D
+# ============================================================================ #
+
+# https://github.com/tomzo/gocd-yaml-config-plugin#setup
+
+# https://docs.gocd.org/current/configuration/configuration_reference.html
+
+---
+format_version: 3
+pipelines:
+ DevOps-Python-tools:
+ group: defaultGroup
+ label_template: ${COUNT}
+ lock_behavior: none
+ display_order: -1
+ materials:
+ git:
+ git: https://github.com/HariSekhon/DevOps-Python-tools
+ shallow_clone: false
+ auto_update: true
+ branch: master
+ stages:
+ - build-and-test:
+ fetch_materials: true
+ keep_artifacts: false
+ clean_workspace: false
+ approval:
+ type: success
+ allow_only_on_success: false
+ jobs:
+ #apt-update:
+ # timeout: 10
+ # tasks:
+ # - exec:
+ # command: apt
+ # arguments:
+ # - update
+ # run_if: passed
+ #install-make:
+ # timeout: 10
+ # tasks:
+ # - exec:
+ # command: apt
+ # arguments:
+ # - install
+ # - -qy
+ # - git
+ # - make
+ # run_if: passed
+ ci-bootstrap:
+ timeout: 10
+ tasks:
+ - exec:
+ command: setup/ci_bootstrap.sh
+ run_if: passed
+ init:
+ timeout: 10
+ tasks:
+ - exec:
+ command: make
+ arguments:
+ - init
+ run_if: passed
+ build:
+ timeout: 60
+ tasks:
+ - exec:
+ command: make
+ arguments:
+ - ci
+ run_if: passed
+ test:
+ timeout: 60
+ tasks:
+ - exec:
+ command: make
+ arguments:
+ - test
+ run_if: passed
diff --git a/cicd/buildspec.yml b/cicd/buildspec.yml
new file mode 100644
index 000000000..f4e887953
--- /dev/null
+++ b/cicd/buildspec.yml
@@ -0,0 +1,66 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-12-19 15:32:28 +0000 (Sat, 19 Dec 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# A W S C o d e B u i l d
+# ============================================================================ #
+
+# References:
+#
+# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
+
+# Operating System should be set to Ubuntu, not Amazon Linux 2
+# - this is both recommended since programming language runtimes are now included in standard image of ubuntu, but also to avoid this error:
+#
+# /usr/bin/amazon-linux-extras
+# /root/.pyenv/versions/3.8.3/bin/python: No module named amazon_linux_extras
+
+version: 0.2
+
+# only on Linux, the user to run as - global setting, alternatively set inside a phase section below for localized user
+#run-as: linux-username
+
+env:
+ shell: bash
+ # don't store sensitive stuff like AWS secret keys in variables, use parameter-store or secrets-manager
+ # any environment variables replace existing environment variables, ie. beware if setting PATH that it'll replace the existing PATH with a non-interpolated literal
+ # project env vars take precedence over these, with start build vars taking highest precedence
+ #variables:
+ # DEBUG: "1"
+ #exported-variables:
+ # - DEBUG
+
+phases:
+ # install prerequisites / languages / frameworks / packages to allow build to work
+ install:
+ #commands:
+ # - setup/ci_bootstrap.sh
+ # languages to install
+ runtime-versions:
+ #java: openjdk11
+ # AWS LTS release of OpenJDK 11
+ java: corretto11
+ golang: 1.14
+ python: 3.8
+ ruby: 2.7
+ # eg. sign in to Amazon ECR or install package dependencies
+ pre_build:
+ commands:
+ - setup/ci_bootstrap.sh
+ build:
+ commands:
+ - echo Build started on `date`
+ - make
+ - echo Build completed on `date`
+ - make test
+ - echo Tests completed on `date`
diff --git a/cicd/cloudbuild.yaml b/cicd/cloudbuild.yaml
new file mode 100644
index 000000000..61e439fa3
--- /dev/null
+++ b/cicd/cloudbuild.yaml
@@ -0,0 +1,41 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-12-19 16:27:26 +0000 (Sat, 19 Dec 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G C P C l o u d B u i l d
+# ============================================================================ #
+
+# References:
+#
+# https://cloud.google.com/cloud-build/docs/build-config
+#
+# https://cloud.google.com/cloud-build/docs/build-debug-locally
+
+
+# gcloud builds submit --config cloudbuild.yaml .
+#
+# cloud-build-local --config cloudbuild.yml --dryrun=false .
+
+# tars $PWD to bucket called ${PROJECT_ID}_cloudbuild
+
+timeout: 3660s
+
+steps:
+ - name: ubuntu:18.04
+ entrypoint: bash
+ args:
+ - '-c'
+ - |
+ setup/ci_bootstrap.sh &&
+ make build test
+ timeout: 3600s
diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py
new file mode 100755
index 000000000..bbd94dbb3
--- /dev/null
+++ b/cloudera_navigator_tables_used.py
@@ -0,0 +1,467 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-09 11:35:47 +0000 (Mon, 09 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Processes Cloudera Navigator API exported CSV logs to list the tables used (SELECT'ed from)
+
+This allows you to see if you're wasting time maintaining datasets nobody is using
+
+Handles more than naive filtering delimited column numbers which will miss many table and database names:
+
+ 1. table/database name fields are often blank and need to be inferred from SQL queries field
+ (currently limited to 'SELECT ... FROM ...' because JOINs are often complicated by use of table aliases)
+ 2. SQL queries often contain newlines which break the rows up - these are recombined in to single records
+ 3. multi-line SQL queries have comments stripped out to avoid false positives of what is being used
+ 4. where table/database field aren't available, also checks if inferrable from resource field
+ to determine database and table name before parsing SQL which is a last resort
+ 5. optionally ignore selected users by regex
+ - matches user or kerberos principal
+ - eg. to omit ETL service account from skewing data access results
+
+Supports reading directly from gzipped logs if they end in .gz file extension.
+However, the gzip library may have issues around universal newline support, if so, gunzip first.
+
+See cloudera_navigator_audit_logs_download.sh for a script to export these logs
+
+./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \\
+ navigator_audit_2020_hive.csv navigator_audit_2020_impala.csv
+
+Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison):
+
+"database","table"
+
+Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10
+(but may require ongoing tweaks depending on quirks in your data set or changes in the API / logs)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import csv
+import gzip
+import logging
+import os
+import re
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_regex, isInt, isPythonMinVersion
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3.0'
+
+
+class ClouderaNavigatorTablesUsed(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(ClouderaNavigatorTablesUsed, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ csv.field_size_limit(sys.maxsize)
+ self.delimiter = None
+ self.quotechar = None
+ self.escapechar = None
+ self.timeout_default = None
+ #self.data = {}
+ self.indicies = {}
+ self.len_headers = None
+ self.table_regex = r'[\w\.`]+'
+ self.re_table = re.compile(self.table_regex)
+ # doesn't handle JOINs because SQL pros usually use table aliases
+ self.re_select_from_table = re.compile(r'\bSELECT\b.+\bFROM\b(?:\s|\n)+({table_regex})'\
+ .format(table_regex=self.table_regex), \
+ re.I | re.MULTILINE | re.DOTALL)
+ ignore_statements = [
+ 'SHOW',
+ 'DESCRIBE',
+ 'USE',
+ 'CREATE',
+ 'DROP',
+ 'INSERT',
+ 'DELETE',
+ 'UPDATE',
+ 'GET_TABLES',
+ 'GET_SCHEMAS',
+ 'VIEW_METADATA',
+ 'ANALYZE',
+ r'COMPUTE\s+STATS',
+ 'REFRESH',
+ r'INVALIDATE\s+METADATA',
+ ]
+ self.re_ignore = re.compile(r'^\s*\b(?:' + '|'.join(ignore_statements) + r')\b',\
+ re.I | re.MULTILINE | re.DOTALL)
+ # 2020-01-31T20:45:59.000Z
+ self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$')
+ self.re_ignored_users = None
+ self.csv_writer = None
+ self.operations_to_ignore = [
+ '',
+ 'HIVEREPLICATIONCOMMAND',
+ 'START',
+ 'STOP',
+ 'RESTART',
+ 'LOAD',
+ 'SWITCHDATABASE',
+ 'USE',
+ ]
+
+ def add_options(self):
+ super(ClouderaNavigatorTablesUsed, self).add_options()
+ # must set type to str otherwise csv module gives this error on Python 2.7:
+ # TypeError: "delimiter" must be string, not unicode
+ # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals'
+ # breaks this - might break in Python 3 if the impyla module doesn't fix behaviour
+ self.add_opt('-d', '--delimiter', default=',', type=str, help='Delimiter to use for outputting (default: ,)')
+ self.add_opt('-Q', '--quotechar', default='"', type=str,
+ help='Generate quoted CSV output (recommended, default is double quote \'"\')')
+ self.add_opt('-E', '--escapechar', help='Escape char if needed (for both reading and writing)')
+ self.add_opt('-i', '--ignore-users', help='Users to ignore, comma separated regex values')
+
+ def process_options(self):
+ super(ClouderaNavigatorTablesUsed, self).process_options()
+ self.delimiter = self.get_opt('delimiter')
+ self.quotechar = self.get_opt('quotechar')
+ self.escapechar = self.get_opt('escapechar')
+ ignore_users = self.get_opt('ignore_users')
+ if ignore_users:
+ ignored_users = ignore_users.split(',')
+ for username in ignored_users:
+ validate_regex(username, 'ignored user')
+ # account for kerberized names - user, user@domain.com or user/host@domain.com
+ self.re_ignored_users = re.compile('^(?:' + '|'.join(ignored_users) + ')(?:[@/]|$)', re.I)
+ if not self.args:
+ self.usage('no CSV file argument given')
+
+ def run(self):
+ quoting = csv.QUOTE_ALL
+ if self.quotechar == '':
+ quoting = csv.QUOTE_NONE
+
+ #fieldnames = ['database', 'table', 'user']
+ fieldnames = ['database', 'table']
+ self.csv_writer = csv.DictWriter(sys.stdout,
+ delimiter=self.delimiter,
+ quotechar=self.quotechar,
+ escapechar=self.escapechar,
+ quoting=quoting,
+ fieldnames=fieldnames)
+ mode = 'rtU'
+ if isPythonMinVersion(3):
+ mode = 'rt'
+ # open(..., encoding="utf-8") is Python 3 only - uses system default otherwise
+ for filename in self.args:
+ if filename.endswith('.gz'):
+ log.debug("processing gzip'd file: %s", filename)
+ with gzip.open(filename, mode) as filehandle:
+ self.process_file(filehandle)
+ else:
+ log.debug("processing file: %s", filename)
+ with open(filename, mode) as filehandle:
+ self.process_file(filehandle)
+
+ #csv_writer.writeheader()
+ #for database in sorted(self.data):
+ # for table in sorted(self.data[database]):
+ # csv_writer.writerow({'database': database,
+ # 'table': table})
+ #if log.isEnabledFor(logging.DEBUG):
+ # sys.stdout.flush()
+
+# Navigator API Audit log output is a mess with duplicate columns and different naming conventions,
+# eg. identical SQL in fields 18 and 36
+# table and database names in fields 19+21 vs 40+41
+# all duplicates with different header names
+#
+# # same result for navigator_audit_2019_impala.csv
+# csv_header_indices.sh navigator_audit_2019_hive.csv
+# 0 Timestamp
+# 1 Username
+# 2 "IP Address"
+# 3 "Service Name"
+# 4 Operation
+# 5 Resource
+# 6 Allowed
+# 7 Impersonator
+# 8 sub_operation
+# 9 entity_id
+# 10 stored_object_name
+# 11 additional_info
+# 12 collection_name
+# 13 solr_version
+# 14 operation_params
+# 15 service
+# 16 operation_text
+# 17 url
+# 18 operation_text
+# 19 table_name
+# 20 resource_path
+# 21 database_name
+# 22 object_type
+# 23 Source
+# 24 Destination
+# 25 Permissions
+# 26 "Delegation Token ID"
+# 27 "Table Name"
+# 28 Family
+# 29 Qualifier
+# 30 "Operation Text"
+# 31 "Database Name"
+# 32 "Table Name"
+# 33 "Object Type"
+# 34 "Resource Path"
+# 35 "Usage Type"
+# 36 "Operation Text"
+# 37 "Query ID"
+# 38 "Session ID"
+# 39 Status
+# 40 "Database Name"
+# 41 "Table Name"
+# 42 "Object Type"
+# 43 Privilege
+
+ def process_file(self, filehandle):
+ csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\')
+ try:
+ # Python 2
+ headers = csv_reader.next()
+ except AttributeError:
+ # Python 3
+ headers = next(csv_reader)
+ self.len_headers = len(headers)
+ # needed to ensure row joining works later on with number of fields left
+ assert self.len_headers == 44
+ user_index = 1
+ operation_index = 4
+ resource_index = 5
+ object_index = 22 # used by collapse_sql_fields to check if SQL was split, do not change to index 33!
+ # --
+ # with massive queries taking the latter 2 is more likely to succeed,
+ # possibly because there is a rare and subtle issue in collapse_sql_fields
+ #table_index = 19
+ #database_index = 21
+ # or
+ table_index = 41
+ database_index = 40
+ # --
+ # fields 18 and 36 are identical SQL - need both to collapse rows later
+ sql_index = 18
+ sql_index2 = 36
+ #assert headers[table_index] == 'table_name' # index 19
+ #assert headers[database_index] == 'database_name' # index 21
+ assert headers[table_index] == 'Table Name' # index 41
+ assert headers[database_index] == 'Database Name' # index 40
+ assert headers[user_index] == 'Username'
+ assert headers[operation_index] == 'Operation'
+ assert headers[resource_index] == 'Resource'
+ assert headers[sql_index] == 'operation_text'
+ assert headers[sql_index2] == 'Operation Text'
+ assert headers[object_index] == 'object_type'
+ self.indicies = {
+ 'user_index': user_index,
+ 'operation_index': operation_index,
+ 'resource_index': resource_index,
+ 'table_index': table_index,
+ 'database_index': database_index,
+ 'sql_index': sql_index,
+ 'sql_index2': sql_index2, # needed for collapsing rows inflated by SQL fragmentation
+ 'object_index': object_index
+ }
+ self.process_rows(csv_reader)
+
+ # logic to reconstruct rows because the Navigator API breaks the record format
+ # with newlines in SQL coming out literally and fragmenting the records
+ def process_rows(self, csv_reader):
+ last_row = []
+ for current_row in csv_reader:
+ #log.debug('current row = %s', current_row)
+ if not current_row:
+ continue
+ if self.is_new_record(current_row):
+ row = last_row
+ last_row = current_row
+ else:
+ last_row += current_row
+ continue
+ if not row:
+ continue
+ self.process_row(row)
+ self.process_row(last_row)
+
+ # originally did this by counting fields but SQL fragmentation generates extra fields
+ def is_new_record(self, current_row):
+ return self.re_timestamp.match(current_row[0])
+
+ def process_row(self, row):
+ if not row:
+ return
+ log.debug('processing row = %s', row)
+ len_row = len(row)
+ log.debug('row len = %s', len_row)
+ if len_row > self.len_headers:
+ row = self.collapse_sql_fields(row=row)
+ len_row = len(row)
+ if len_row != self.len_headers:
+ raise AssertionError('row items ({}) != header items ({}) for offending row: {}'\
+ .format(len_row, self.len_headers, row))
+ (database, table) = self.parse_table(row)
+ self.output(row=row, database=database, table=table)
+
+ def parse_table(self, row):
+ #log.debug(row)
+ user = row[self.indicies['user_index']]
+ # user: 'hari.sekhon'
+ # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com'
+ if self.re_ignored_users and self.re_ignored_users.match(user):
+ log.debug('skipping row for ignored user %s: %s', user, row)
+ return (None, None)
+ database = row[self.indicies['database_index']].strip()
+ table = row[self.indicies['table_index']].strip()
+ if not database or not table or not self.re_table.match('{}.{}'.format(database, table)):
+ #log.info('table not found in fields for row: %s', row)
+ operation = row[self.indicies['operation_index']]
+ if operation in self.operations_to_ignore:
+ return (None, None)
+ elif operation == 'QUERY':
+ query = row[self.indicies['sql_index']]
+ # cheaper than re_ignore to pre-filter
+ if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'):
+ return (None, None)
+ (database, table) = self.get_db_table_from_resource(row)
+ if database and table:
+ pass
+ else:
+ log.debug('database/table not found in row: %s', row)
+ log.debug('trying to parse: %s', query)
+ match = self.re_select_from_table.search(query)
+ if match:
+ table = match.group(1)
+ if '.' in table:
+ (database, table) = table.split('.', 1)
+ # could use .search but all these seem to be at beginning
+ elif self.re_ignore.match(query):
+ return (None, None)
+ else:
+ log.warning('failed to parse database/table from query: %s', query)
+ return (None, None)
+ else:
+ log.debug('database/table not found in row and operation is not a query to parse: %s', row)
+ return (None, None)
+ if not table and not database:
+ return (None, None)
+ if table:
+ table = table.lower().strip('`')
+ if ' ' in table:
+ raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\
+ .format(table, self.index_output(row)))
+ if database:
+ database = database.lower().strip('`')
+ if ' ' in database:
+ raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\
+ .format(database, self.index_output(row)))
+ if table == 'null':
+ raise AssertionError('table == null - parsing error for row: {}'.format(row))
+ return (database, table)
+
+ def get_db_table_from_resource(self, row):
+ database = None
+ table = None
+ resource = row[self.indicies['resource_index']]
+ if resource and \
+ ':' in resource and \
+ 'null:null' not in resource:
+ # database:table in Resource field
+ (database, table) = resource.split(':', 1)
+ return (database, table)
+
+ def output(self, row, database, table):
+ if not self.re_table.match('{}.{}'.format(database, table)):
+ log.warning('%s.%s does not match table regex', database, table)
+ return
+ # instead of collecting in ram, now just post-process through sort -u
+ # this way it is easier to see live extractions, --debug and correlate
+ #self.data[database] = self.data.get(database, {})
+ #self.data[database][table] = 1
+ if table and not database:
+ log.info('got table but not database for row: %s', row)
+ if database and not table:
+ log.info('got database but not table for row: %s', row)
+ if not table and not database:
+ return
+ #self.csv_writer.writerow({'database': database, 'table': table, 'user': row[self.indicies['user_index']]})
+ self.csv_writer.writerow({'database': database, 'table': table})
+ if log.isEnabledFor(logging.DEBUG):
+ sys.stdout.flush()
+
+ def collapse_sql_fields(self, row):
+ sql_index = self.indicies['sql_index']
+ sql_index2 = self.indicies['sql_index2']
+ object_index = self.indicies['object_index']
+ len_row = len(row)
+ if len_row > self.len_headers:
+ log.debug('collapsing fields in row: %s', row)
+ # divide by 2 to account for this having been done twice in duplicated SQL operational text
+ # Update: appears this broke as only 2nd occurence of SQL operational text field got split to new fields,
+ # which is weird because the log shows both 1st and 2nd SQL text fields were double quoted
+ difference = len_row - self.len_headers
+ # seems first occurrence doesn't get split in some occurence,
+ # wasn't related to open in newline universal mode though
+ # if 2 fields after isn't the /user/hive/warehouse/blah.db then 1st SQL wasn't split
+ # would have to regex /user/hive/warehouse/blah.db(?:/table)?
+ #if not row[sql_index+2].endswith('.db'):
+ # if object field is TABLE or DATABASE then 1st sql field wasn't split
+ if row[object_index] not in ('TABLE', 'DATABASE'):
+ difference /= 2
+ # slice indicies must be integers
+ if not isInt(difference):
+ raise AssertionError("difference in field length '{}' is not an integer for row: {}"\
+ .format(difference, row))
+ difference = int(difference)
+ row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]])
+ row = row[:sql_index] + row[sql_index+difference:]
+ row[sql_index2] = ','.join([self.sql_decomment(_) for _ in row[sql_index2:difference]])
+ row = row[:sql_index2] + row[sql_index2+difference:]
+ log.debug('collapsed row: %s', row)
+ else:
+ log.debug('not collapsing row: %s', row)
+ return row
+
+ @staticmethod
+ def sql_decomment(string):
+ return string.split('--')[0].strip()
+
+ @staticmethod
+ def index_output(obj):
+ return '\n'.join(['{}\t{}'.format(index, item) for (index, item) in enumerate(obj)])
+
+
+if __name__ == '__main__':
+ ClouderaNavigatorTablesUsed().main()
diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py
new file mode 100755
index 000000000..972716236
--- /dev/null
+++ b/cloudera_navigator_tables_used_postgres.py
@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-16 19:21:24 +0000 (Mon, 16 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Processes Cloudera Navigator CSV logs exported from PostgreSQL to list the tables used (SELECT'ed from)
+
+This allows you to see if you're wasting time maintaining datasets nobody is using
+
+See cloudera_navigator_audit_logs_export_postgres.sh for a script to export these logs
+
+Supports reading directly from gzipped logs if they end in .gz file extension.
+However, the gzip library may have issues around universal newline support, if so, gunzip first.
+
+./cloudera_navigator_tables_used_postgres.py nav.public.hive_audit_events_*.csv.gz \\
+ nav.public.impala_audit_events_*.csv.gz ...
+
+Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison):
+
+"database","table"
+
+Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import csv
+#import logging
+import os
+import re
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, isInt
+ #from harisekhon import CLI
+ from cloudera_navigator_tables_used import ClouderaNavigatorTablesUsed
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.2.1'
+
+
+class ClouderaNavigatorTablesUsedPostgreSQL(ClouderaNavigatorTablesUsed):
+
+ def __init__(self):
+ # Python 2.x
+ super(ClouderaNavigatorTablesUsedPostgreSQL, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ # recombine records due to SQL \n breaking up records, new records start like:
+ # 306529,1574163624392,1,hive,
+ self.re_new_record = re.compile(r'^\d+,\d+,[01],(?:hive|impala),')
+ # get db + table from resource path (just one layer of checks)
+ self.re_resource = re.compile(r'/(\w+)\.db/(\w+)')
+
+# Navigator table logs:
+#
+# gzcat nav.public.hive_audit_events_2019_11_19.csv.gz | csv_header_indices.sh
+# 0 id
+# 1 event_time
+# 2 allowed
+# 3 service_name
+# 4 username
+# 5 ip_addr
+# 6 operation
+# 7 database_name
+# 8 object_type
+# 9 table_name
+# 10 operation_text
+# 11 impersonator
+# 12 resource_path
+# 13 object_usage_type
+
+# gzcat nav.public.impala_audit_events_2019_11_19.csv.gz | csv_header_indices.sh
+# 0 id
+# 1 event_time
+# 2 allowed
+# 3 service_name
+# 4 username
+# 5 impersonator
+# 6 ip_addr
+# 7 operation
+# 8 query_id
+# 9 session_id
+# 10 status
+# 11 database_name
+# 12 object_type
+# 13 table_name
+# 14 privilege
+# 15 operation_text
+
+ def process_file(self, filehandle):
+ csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\')
+ headers = csv_reader.next()
+ self.len_headers = len(headers)
+ # needed to ensure row joining works later on with number of fields left
+ assert self.len_headers == 14 or self.len_headers == 16
+ user_index = 4
+ assert headers[user_index] == 'username'
+ # Hive postgres audit log
+ if self.len_headers == 14:
+ operation_index = 6
+ database_index = 7
+ table_index = 9
+ sql_index = 10
+ resource_index = 12
+ assert headers[resource_index] == 'resource_path'
+ # Impala postgres audit log
+ elif self.len_headers == 16:
+ operation_index = 7
+ database_index = 11
+ table_index = 13
+ sql_index = 15
+ resource_index = None
+ else:
+ raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala')
+ assert headers[sql_index] == 'operation_text'
+ assert headers[database_index] == 'database_name'
+ assert headers[table_index] == 'table_name'
+ assert headers[operation_index] == 'operation'
+ self.indicies = {
+ 'user_index': user_index,
+ 'operation_index': operation_index,
+ 'resource_index': resource_index,
+ 'table_index': table_index,
+ 'database_index': database_index,
+ 'sql_index': sql_index,
+ }
+ self.process_rows(csv_reader)
+
+ def is_new_record(self, current_row):
+ return self.re_new_record.match(','.join(current_row))
+
+ def parse_table(self, row):
+ #log.debug(row)
+ user = row[self.indicies['user_index']]
+ # user: 'hari.sekhon'
+ # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com'
+ if self.re_ignored_users and self.re_ignored_users.match(user):
+ log.debug('skipping row for ignored user %s: %s', user, row)
+ return (None, None)
+ database = row[self.indicies['database_index']].strip()
+ table = row[self.indicies['table_index']].strip()
+ if not database or not table or not self.re_table.match('{}.{}'.format(database, table)):
+ #log.info('table not found in fields for row: %s', row)
+ operation = row[self.indicies['operation_index']]
+ if operation in self.operations_to_ignore:
+ return (None, None)
+ elif operation == 'QUERY':
+ query = row[self.indicies['sql_index']]
+ # cheaper than re_ignore to pre-filter
+ if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'):
+ return (None, None)
+ (database, table) = self.get_db_table_from_resource(row)
+ if database and table:
+ pass
+ else:
+ log.debug('database/table not found in row: %s', row)
+ log.debug('trying to parse: %s', query)
+ match = self.re_select_from_table.search(query)
+ if match:
+ table = match.group(1)
+ if '.' in table:
+ (database, table) = table.split('.', 1)
+ # could use .search but all these seem to be at beginning
+ elif self.re_ignore.match(query):
+ return (None, None)
+ else:
+ log.warning('failed to parse database/table from query: %s', query)
+ return (None, None)
+ else:
+ log.debug('database/table not found in row and operation is not a query to parse: %s', row)
+ return (None, None)
+ if not table and not database:
+ return (None, None)
+ if table:
+ table = table.lower().strip('`')
+ if ' ' in table:
+ raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\
+ .format(table, self.index_output(row)))
+ if database:
+ database = database.lower().strip('`')
+ if ' ' in database:
+ raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\
+ .format(database, self.index_output(row)))
+ if table == 'null':
+ raise AssertionError('table == null - parsing error for row: {}'.format(row))
+ return (database, table)
+
+ def get_db_table_from_resource(self, row):
+ # only available for hive audit logs, not impala
+ if self.indicies['resource_index'] is None:
+ return (None, None)
+ database = None
+ table = None
+ resource = row[self.indicies['resource_index']]
+ if resource:
+ match = self.re_resource.search(resource)
+ if match:
+ database = match.group(1)
+ table = match.group(2)
+ return (database, table)
+
+ def collapse_sql_fields(self, row):
+ sql_index = self.indicies['sql_index']
+ len_row = len(row)
+ if len_row > self.len_headers:
+ log.debug('collapsing fields in row: %s', row)
+ difference = len_row - self.len_headers
+ # slice indicies must be integers
+ if not isInt(difference):
+ raise AssertionError("difference in field length '{}' is not an integer for row: {}"\
+ .format(difference, row))
+ difference = int(difference)
+ row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]])
+ row = row[:sql_index] + row[sql_index+difference:]
+ log.debug('collapsed row: %s', row)
+ else:
+ log.debug('not collapsing row: %s', row)
+ return row
+
+
+if __name__ == '__main__':
+ ClouderaNavigatorTablesUsedPostgreSQL().main()
diff --git a/codefresh.yml b/codefresh.yml
new file mode 100644
index 000000000..7cddcaf7c
--- /dev/null
+++ b/codefresh.yml
@@ -0,0 +1,43 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-24 17:43:07 +0000 (Mon, 24 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C o d e f r e s h C I
+# ============================================================================ #
+
+# https://codefresh.io/docs/docs/codefresh-yaml/
+
+version: "1.0"
+stages:
+ - "checkout"
+ - "build"
+steps:
+ checkout:
+ type: "git-clone"
+ description: "Cloning main repository..."
+ repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}'
+ revision: "${{CF_REVISION}}"
+ stage: "checkout"
+ build:
+ title: Running docker image
+ type: freestyle
+ working_directory: '${{CF_REPO_NAME}}'
+ arguments:
+ image: 'ubuntu:18.04'
+ commands:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py
new file mode 100755
index 000000000..2e675c253
--- /dev/null
+++ b/crunch_accounting_csv_statement_converter.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-02-04 23:24:30 +0000 (Mon, 04 Feb 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to convert Bank or Credit Card CSV Statements with headers to Crunch Accounting standard format
+for importing to Crunch for accounts reconciliation
+
+Extracts the important fields, calculates the running balance if necessary
+
+File arguments are read one by one and converted and output to a file of the same name with _crunch.csv at the end
+
+If there is no balance column you must provide a --starting-balance from which to calculate the running balance column
+for Crunch, In that case if specifying multiple CSV statements as arguments, they must be given in chronological order
+in order for one statement's running balance to flow on to the next in the correct order
+
+Can easily be extended for other Banks CSV formats, as it's just simple matching of the headers
+
+For additional formats just raise an github issue request with a sample CSV
+
+Tested on Barclaycard Commercial statement CSV exports
+(Barclaycard lists entries in reverse chronological order so specify --reverse-order as well as --credit-card)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import csv
+import os
+import re
+import sys
+import tempfile
+import traceback
+from decimal import Decimal
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, isChars
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.7.4'
+
+
+class CrunchAccountingCsvStatementConverter(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(CrunchAccountingCsvStatementConverter, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.verbose = 2
+ self.running_balance = None
+ self.default_timeout = None
+ self.credit_card = False
+ self.reverse_order = False
+
+ def add_options(self):
+ super(CrunchAccountingCsvStatementConverter, self).add_options()
+ self.add_opt('-c', '--credit-card', action='store_true',
+ help='Credit Card statements (inverts transactions against running balance)')
+ self.add_opt('-r', '--reverse-order', action='store_true',
+ help='Statement entries are in reverse chronological order (eg. Barclaycard)')
+ self.add_opt('-s', '--starting-balance', help='Starting Balance (used if no balance column is found')
+
+ def process_options(self):
+ super(CrunchAccountingCsvStatementConverter, self).process_options()
+ self.credit_card = self.get_opt('credit_card')
+ self.reverse_order = self.get_opt('reverse_order')
+ self.running_balance = self.get_opt('starting_balance')
+ if self.running_balance is not None:
+ try:
+ self.running_balance = Decimal(self.running_balance)
+ except ValueError as _:
+ log.error('INVALID starting balance %s, must be in a decimal number: %s', self.running_balance, _)
+ sys.exit(1)
+
+ def run(self):
+ if not self.args:
+ self.usage('no file arguments specified')
+ for filename in self.args:
+ target_filename = '{}_crunch.csv'.format(re.sub(r'\.csv$', '', filename))
+ log.info("converting file '%s' => '%s'", filename, target_filename)
+ if self.convert(filename, target_filename):
+ log.info("converted '%s' => '%s'", filename, target_filename)
+ else:
+ log.error("FAILED to convert filename '%s'", filename)
+ sys.exit(2)
+ log.info('Final Balance: {}'.format(self.running_balance))
+
+ def convert(self, filename, target_filename):
+ if self.reverse_order:
+ filename = self.reverse_contents(filename)
+ csvreader = self.get_csvreader(filename)
+ if not csvreader:
+ return False
+ count = 0
+ (positions, balance_position) = self.detect_columns(csvreader)
+ csvwriter = csv.writer(open(target_filename, 'w'))
+ csvwriter.writerow(['Date', 'Description', 'Amount', 'Balance'])
+ for row in csvreader:
+ count += 1
+ amount = self.amount(row[positions['amount']])
+ if balance_position is not None:
+ balance = row[balance_position]
+ elif self.running_balance is not None:
+ self.running_balance += amount
+ balance = self.running_balance
+ else:
+ log.error('no balance column found and no running balance given')
+ sys.exit(2)
+ csvwriter.writerow(
+ [
+ row[positions['date']],
+ row[positions['desc']],
+ amount,
+ balance
+ ]
+ )
+ log.info('%s CSV lines processed', count)
+ return True
+
+ def amount(self, amount):
+ if self.credit_card:
+ return -Decimal(amount)
+ return Decimal(amount)
+
+ @staticmethod
+ def reverse_contents(filename):
+ lines = open(filename).readlines()
+ lines = [lines[0]] + list(reversed(lines[1:]))
+ (_, tmp_filename) = tempfile.mkstemp(text=True)
+ filehandle = open(tmp_filename, 'w')
+ for line in lines:
+ filehandle.write(line)
+ return tmp_filename
+
+ def detect_columns(self, csvreader):
+ headers = next(csvreader)
+ if headers[0][0] == '{':
+ log.error('JSON opening braces detected, not a CSV?')
+ return False
+ positions = {'date': None, 'desc': None, 'amount': None}
+ balance_position = None
+ for (position, value) in enumerate(headers):
+ # want Transaction Date and not Posted Date
+ if 'Date' in value and not 'Posted' in value:
+ positions['date'] = position
+ # Original Amount column will be original currency eg 499 USD, but we only want native currency eg. 421.33
+ elif 'Amount' in value and not 'Original' in value:
+ positions['amount'] = position
+ elif 'Balance' in value:
+ balance_position = position
+ # Barclaycard CSVs
+ elif 'Merchant Name' in value:
+ positions['desc'] = position
+ # Barclays CSVs
+ elif 'Memo' in value:
+ positions['desc'] = position
+ for pos in positions:
+ if positions[pos] is None:
+ log.error('field %s not found', pos)
+ sys.exit(1)
+ if balance_position is None and self.running_balance is None:
+ self.usage('no balance column detected, please specify --starting-balance')
+ return (positions, balance_position)
+
+ @staticmethod
+ def get_csvreader(filename):
+ try:
+ filehandle = open(filename)
+ except IOError as _:
+ log.error(_)
+ return None
+ filename = os.path.basename(filename)
+ try:
+ dialect = csv.Sniffer().sniff(filehandle.read(1024))
+ # this will raise an Error if invalid
+ dialect.strict = True
+ filehandle.seek(0)
+ csvreader = csv.reader(filehandle, dialect)
+ except csv.Error as _:
+ log.warning('file %s: %s', filename, _)
+ # in Python 2 must be string not unicode
+ csvreader = csv.reader(filehandle, delimiter=str(','), quotechar=None)
+ csvreader = CrunchAccountingCsvStatementConverter.validate_csvreader(csvreader, filename)
+ filehandle.seek(0)
+ return csvreader
+
+ @staticmethod
+ def validate_csvreader(csvreader, filename):
+ count = 0
+ try:
+ # csvreader doesn't seem to generate any errors ever :-(
+ # csv module allows entire lines of json/xml/yaml to go in as a single field
+ # Adding some invalidations manually
+ for field_list in csvreader:
+ # list of fields with no separator information
+ log.debug("line: %s", field_list)
+ # make it fail if there is only a single field on any line
+ if len(field_list) < 3:
+ log.error("less than 3 fields detected, aborting conversion of file '%s'", filename)
+ return None
+ # extra protection along the same lines as anti-json:
+ # the first char of field should be alphanumeric, not syntax
+ # however instead of isAlnum allow quotes for quoted CSVs to pass validation
+ if field_list[0] not in ("", " ") and not isChars(field_list[0][0], 'A-Za-z0-9"'):
+ log.warning('non-alphanumeric / quote opening character detected in CSV first field' + \
+ '"{}"'.format(field_list[0]))
+ #return None
+ count += 1
+ except csv.Error as _:
+ log.warning('file %s, line %s: %s', filename, csvreader.line_num, _)
+ return None
+ if count == 0:
+ log.error('zero lines detected, blank input is not valid CSV')
+ return None
+ return csvreader
+
+
+if __name__ == '__main__':
+ CrunchAccountingCsvStatementConverter().main()
diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh
new file mode 100755
index 000000000..61a1235bd
--- /dev/null
+++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1090
+. "$srcdir/lib.sh"
+
+# statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv
+export STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv"
+
+generate_crunch_statements --credit-card --reverse-order
diff --git a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh
new file mode 100755
index 000000000..f708fec0c
--- /dev/null
+++ b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1090
+. "$srcdir/lib.sh"
+
+# statements should be named in format: Barclays_Statement_YYYY-MM-DD.csv
+export STATEMENT_GLOB="Barclays_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv"
+
+# Barclays CSV statements often have whitespace starting fields instead of blank or 'null'
+# unfortunately this resets all the original CSV timestamps each run so it's best to do only where needed
+# UPDATE: no longer necessary, converter just ignores these blank fields now in the validation
+#for statement in $STATEMENT_GLOB; do
+ #perl -pi -e 's/^\s+,/,/' "$statement"
+#done
+
+generate_crunch_statements --reverse-order
diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh
new file mode 100755
index 000000000..e77e66834
--- /dev/null
+++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck disable=SC1090
+#. "$srcdir/lib/utils.sh"
+
+converter="$srcdir/../crunch_accounting_csv_statement_converter.py"
+
+get_latest_crunch_statement(){
+ local latest_crunch_statement
+ for statement in $STATEMENT_GLOB; do
+ crunch_statement="${statement%.csv}_crunch.csv"
+ if [ -f "$crunch_statement" ]; then
+ latest_crunch_statement="$crunch_statement"
+ fi
+ done
+ if [ -n "${latest_crunch_statement:-}" ]; then
+ echo "$latest_crunch_statement"
+ fi
+}
+
+get_final_balance_from_statement(){
+ local crunch_statement="$1"
+ if ! [[ "$crunch_statement" =~ _crunch.csv$ ]]; then
+ echo "invalid statement passed to get_final_balance_from_statement(), must be *_crunch.csv" >&2
+ exit 1
+ fi
+ tail -n 1 "$crunch_statement" | awk -F, '{print $4}'
+}
+
+get_starting_balance(){
+ local starting_balance
+ local latest_crunch_statement="$1"
+ if [ -n "${latest_crunch_statement:-}" ]; then
+ echo "latest crunch statement is $latest_crunch_statement" >&2
+ starting_balance="$(get_final_balance_from_statement "$latest_crunch_statement")"
+ else
+ echo "no latest crunch statement, getting starting balance from environment variable \$STARTING_BALANCE" >&2
+ starting_balance="${STARTING_BALANCE:-}"
+ if [ -z "$starting_balance" ]; then
+ echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$STARTING_BALANCE" >&2
+ exit 1
+ fi
+ fi
+ echo "starting balance: $starting_balance" >&2
+ echo "$starting_balance"
+}
+
+generate_crunch_statements(){
+ # only generate statements newer than the last generated one which provides the starting balance
+ local passed_latest_statement=0
+ local latest_crunch_statement
+ local starting_balance
+ latest_crunch_statement="$(get_latest_crunch_statement)"
+ if [ -z "$latest_crunch_statement" ]; then
+ passed_latest_statement=1
+ fi
+ starting_balance="$(get_starting_balance "$latest_crunch_statement")"
+ for statement in $STATEMENT_GLOB; do
+ crunch_statement="${statement%.csv}_crunch.csv"
+ if [ -f "$crunch_statement" ]; then
+ if [ $passed_latest_statement = 0 ] &&
+ [ "$crunch_statement" = "$latest_crunch_statement" ]; then
+ passed_latest_statement=1
+ fi
+ continue
+ fi
+ if [ $passed_latest_statement -lt 1 ]; then
+ continue
+ fi
+ "$converter" "$@" --starting-balance "$starting_balance" "$statement"
+ starting_balance="$(get_final_balance_from_statement "$crunch_statement")"
+ done
+}
diff --git a/docker_pull_all_images.sh b/docker_pull_all_images.sh
index 44fd78f66..e6d39fd0d 100755
--- a/docker_pull_all_images.sh
+++ b/docker_pull_all_images.sh
@@ -4,19 +4,19 @@
# Author: Hari Sekhon
# Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir="$(dirname $0)"
+srcdir="$(dirname "$0")"
for docker_image in $("$srcdir/dockerhub_search.py" -q harisekhon); do
echo "docker pull $docker_image"
diff --git a/docker_pull_all_images_all_tags.sh b/docker_pull_all_images_all_tags.sh
index 5276ff6dd..d12c3e25b 100755
--- a/docker_pull_all_images_all_tags.sh
+++ b/docker_pull_all_images_all_tags.sh
@@ -4,19 +4,19 @@
# Author: Hari Sekhon
# Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir="$(dirname $0)"
+srcdir="$(dirname "$0")"
for docker_image in $("$srcdir/dockerhub_search.py" -q harisekhon | grep -v 'presto.*dev'); do
echo "pulling all tags for docker image $docker_image:"
diff --git a/docker_pull_all_tags.sh b/docker_pull_all_tags.sh
index ade394ef3..229f3fe68 100755
--- a/docker_pull_all_tags.sh
+++ b/docker_pull_all_tags.sh
@@ -4,19 +4,19 @@
# Author: Hari Sekhon
# Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir="$(dirname $0)"
+srcdir="$(dirname "$0")"
docker_image="${1:-}"
diff --git a/docker_registry_show_tags.py b/docker_registry_show_tags.py
index 50d1af743..8c18d3cdc 100755
--- a/docker_registry_show_tags.py
+++ b/docker_registry_show_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/dockerfiles_check_git_branches.py b/dockerfiles_check_git_branches.py
index a7edf5195..662dce3e2 100755
--- a/dockerfiles_check_git_branches.py
+++ b/dockerfiles_check_git_branches.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-20 20:24:12 +0100 (Fri, 20 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
r"""
@@ -91,7 +91,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.7.2'
+__version__ = '0.7.3'
class DockerfileGitBranchCheckTool(CLI):
@@ -321,8 +321,8 @@ def check_file(self, filename, branch):
def check_dockerfile_arg(self, filename, branch):
log.debug('check_dockerfile_arg({0}, {1})'.format(filename, branch))
- branch_base = str(branch).replace('-dev', '')
- (branch_base, branch_versions) = self.branch_version(branch)
+ branch_stripped = str(branch).replace('-dev', '')
+ (branch_base, branch_versions) = self.branch_version(branch_stripped)
with open(filename) as filehandle:
version_index = 0
for line in filehandle:
diff --git a/dockerfiles_check_git_tags.py b/dockerfiles_check_git_tags.py
index 996f787b8..3e0b627bd 100755
--- a/dockerfiles_check_git_tags.py
+++ b/dockerfiles_check_git_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-20 20:24:12 +0100 (Fri, 20 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
r"""
diff --git a/dockerhub_search.py b/dockerhub_search.py
index cb226e59f..ba7034145 100755
--- a/dockerhub_search.py
+++ b/dockerhub_search.py
@@ -1,17 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
+# args: harisekhon
#
# Author: Hari Sekhon
# Date: 2016-05-27 13:15:30 +0100 (Fri, 27 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -20,12 +21,20 @@
Mimics 'docker search' results format but more flexible
-Docker CLI doesn't currently support configuring the returned number of search results and always returns 25:
+Older Docker CLI didn't support configuring the returned number of search results and always returned 25:
https://github.com/docker/docker/issues/23055
Verbose mode will also show a summary for number of results displayed and total number of results available
+Caveat: maxes out at 100 results, to iterate for more than that see dockerhub_search.sh
+
+See also:
+
+ dockerhub_search.sh
+
+in the DevOps Bash tools repo - https://github.com/HariSekhon/DevOps-Python-tools
+
"""
from __future__ import absolute_import
@@ -56,7 +65,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.6'
+__version__ = '0.6.2'
class DockerHubSearch(CLI):
@@ -71,7 +80,7 @@ def __init__(self):
self.quiet = False
def add_options(self):
- self.add_opt('-n', '--num', '--limit', default=50,
+ self.add_opt('-l', '--limit', default=50, type=int,
help='Number of results to return (default: 50)')
self.add_opt('-q', '--quiet', action='store_true',
help='Output only the image names, one per line (useful for shell scripting)')
@@ -84,9 +93,9 @@ def run(self):
self.quiet = self.get_opt('quiet')
term = self.args[0]
log.info('term: %s', term)
- num = self.get_opt('num')
- validate_int(num, 'limit', 1, 1000)
- self.print_results(self.args[0], num)
+ limit = self.get_opt('limit')
+ validate_int(limit, 'limit', 1, 1000)
+ self.print_results(self.args[0], limit)
def print_results(self, term, limit=None):
data = self.search(term, limit)
diff --git a/dockerhub_show_tags.py b/dockerhub_show_tags.py
index 88c9ddc08..59cd6cb5f 100755
--- a/dockerhub_show_tags.py
+++ b/dockerhub_show_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -32,6 +32,7 @@
import json
import logging
import os
+import re
import sys
import traceback
import urllib
@@ -52,7 +53,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.6.2'
+__version__ = '0.6.3'
class DockerHubTags(CLI):
@@ -79,8 +80,11 @@ def run(self):
self.usage('no repos given as args')
self.quiet = self.get_opt('quiet')
if not self.quiet:
+ # cheaper but lgtm hassling me, not a security issue but will shut them up
print('\nDocker', end='')
- if 'registry.hub.docker.com' in self.url_base:
+ #if 'registry.hub.docker.com' in self.url_base:
+ # match anchors but I prefer explicit anchor, more intuitive for other generic language coders
+ if re.match(r'^https://registry\.hub\.docker\.com/', self.url_base):
print('Hub')
else:
print(' Registry: {0}'.format(self.url_base.split('/v2', 1)[0]))
diff --git a/find_active_apache_drill.py b/find_active_apache_drill.py
index 92a07bddb..1e89fb676 100755
--- a/find_active_apache_drill.py
+++ b/find_active_apache_drill.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -32,7 +32,7 @@
Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print
NO_AVAILABLE_SERVER.
-Tested on Apache Drill 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 1.13
+Tested on Apache Drill 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 1.13, 1.14, 1.15
"""
diff --git a/find_active_apache_drill2.py b/find_active_apache_drill2.py
index aaef3ad2a..364b0eda3 100755
--- a/find_active_apache_drill2.py
+++ b/find_active_apache_drill2.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -32,7 +32,7 @@
Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print
NO_AVAILABLE_SERVER.
-Tested on Apache Drill 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 1.13
+Tested on Apache Drill 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 1.13, 1.14, 1.15
(API endpoint is not available in versions <= 1.6, see find_active_apache_drill.py instead)
diff --git a/find_active_cassandra.py b/find_active_cassandra.py
new file mode 100755
index 000000000..c16ee217c
--- /dev/null
+++ b/find_active_cassandra.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: Wed Sep 13 13:58:21 CEST 2017
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to return the first active Cassandra node from an argument list of hosts
+
+Can mix and match between a comma separated list of hosts (--host server1,server2 or contents of the $HOST
+environment variable if not specified) and general free-form space separated arguments, which is useful if piping
+a host list through xargs.
+
+Multi-threaded for speed and exits upon first available host response to minimize delay to ~ 1 second or less.
+
+Useful for simplying scripting or generically extending tools that don't support HBase High Availability directly
+
+By default checks the same --port on all servers. Hosts may have optional : suffixes added to individually
+override each one.
+
+Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print
+NO_AVAILABLE_SERVER.
+
+Tested on Cassandra 3.11
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import os
+import sys
+import traceback
+#from random import shuffle
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from find_active_server import FindActiveServer
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.7.1'
+
+
+class FindActiveCassandra(FindActiveServer):
+
+ def __init__(self):
+ # Python 2.x
+ super(FindActiveCassandra, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ # TCP only check - JMX port, could just as easily use CQL 9042
+ self.default_port = 7199
+ self.protocol = None
+
+ def add_options(self):
+ self.add_hostoption(name='Cassandra', default_port=self.default_port)
+ #self.add_ssl_opt()
+ self.add_common_opts()
+
+ def process_options(self):
+ self.validate_common_opts()
+
+
+if __name__ == '__main__':
+ FindActiveCassandra().main()
diff --git a/find_active_elasticsearch.py b/find_active_elasticsearch.py
index 42d98d4b8..be449bc79 100755
--- a/find_active_elasticsearch.py
+++ b/find_active_elasticsearch.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hadoop2_namenode.py b/find_active_hadoop2_namenode.py
new file mode 100755
index 000000000..7aad7e77d
--- /dev/null
+++ b/find_active_hadoop2_namenode.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: Tue Sep 5 10:49:49 CEST 2017
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to return the active Hadoop Namenode from an argument list of hosts
+
+Can mix and match between a comma separated list of hosts (--host server1,server2 or contents of the $HOST
+environment variable if not specified) and general free-form space separated arguments, which is useful if piping
+a host list through xargs.
+
+Multi-threaded for speed and exits upon first available host response to minimize delay to ~ 1 second or less.
+
+Useful for simplying scripting or generically extending tools that predate Namenode High Availability
+
+By default checks the same --port on all servers. Hosts may have optional : suffixes added to individually
+override each one.
+
+Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print
+NO_AVAILABLE_SERVER.
+
+Tested on Hadoop 2.7.3 on HDP 2.6.1 and Apache Hadoop 2.3, 2.4, 2.5, 2.6, 2.7, 2.8
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import os
+import sys
+import traceback
+#from random import shuffle
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from find_active_hadoop_namenode import FindActiveHadoopNamenode
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.7.2'
+
+
+class FindActiveHadoop2Namenode(FindActiveHadoopNamenode):
+
+ def __init__(self):
+ # Python 2.x
+ super(FindActiveHadoop2Namenode, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.default_port = 50070
+
+
+if __name__ == '__main__':
+ FindActiveHadoop2Namenode().main()
diff --git a/find_active_hadoop_namenode.py b/find_active_hadoop_namenode.py
index 6e6c668ad..1f6f2d44d 100755
--- a/find_active_hadoop_namenode.py
+++ b/find_active_hadoop_namenode.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -32,7 +32,9 @@
Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print
NO_AVAILABLE_SERVER.
-Tested on Hadoop 2.7.3 on HDP 2.6.1 and Apache Hadoop 2.3, 2.4, 2.5, 2.6, 2.7, 2.8
+Tested on Hadoop 3.0, 3.1
+
+For older Hadoop versions see adjacent find_active_hadoop2_namenode.py
"""
@@ -56,7 +58,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.7.2'
+__version__ = '0.8.0'
class FindActiveHadoopNamenode(FindActiveServer):
@@ -66,7 +68,7 @@ def __init__(self):
super(FindActiveHadoopNamenode, self).__init__()
# Python 3.x
# super().__init__()
- self.default_port = 50070
+ self.default_port = 9870
self.protocol = 'http'
self.url_path = '/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus'
self.regex = r'"State"\s*:\s*"active"'
diff --git a/find_active_hadoop_yarn_resource_manager.py b/find_active_hadoop_yarn_resource_manager.py
index d1179c7cc..f2f0e48e3 100755
--- a/find_active_hadoop_yarn_resource_manager.py
+++ b/find_active_hadoop_yarn_resource_manager.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 6 14:44:38 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hbase_master.py b/find_active_hbase_master.py
index e89c87613..cc7de0222 100755
--- a/find_active_hbase_master.py
+++ b/find_active_hbase_master.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hbase_stargate.py b/find_active_hbase_stargate.py
index c62b1756c..79dc122fa 100755
--- a/find_active_hbase_stargate.py
+++ b/find_active_hbase_stargate.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hbase_thrift.py b/find_active_hbase_thrift.py
index a83594730..06e44a414 100755
--- a/find_active_hbase_thrift.py
+++ b/find_active_hbase_thrift.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_impala.py b/find_active_impala.py
index cce6eb399..ae086f46a 100755
--- a/find_active_impala.py
+++ b/find_active_impala.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_impala_catalog.py b/find_active_impala_catalog.py
index 7fc7a143e..cb6ea0de3 100755
--- a/find_active_impala_catalog.py
+++ b/find_active_impala_catalog.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_impala_statestore.py b/find_active_impala_statestore.py
index 3ec124494..049ed78f5 100755
--- a/find_active_impala_statestore.py
+++ b/find_active_impala_statestore.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_kubernetes_api.py b/find_active_kubernetes_api.py
new file mode 100755
index 000000000..a93b0bc81
--- /dev/null
+++ b/find_active_kubernetes_api.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: Wed Sep 13 13:58:21 CEST 2017
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to return the the active Kubernetes API server from an argument list of hosts
+
+Can mix and match between a comma separated list of hosts (--host server1,server2 or contents of the $HOST
+environment variable if not specified) and general free-form space separated arguments, which is useful if piping
+a host list through xargs.
+
+Multi-threaded for speed and exits upon first available host response to minimize delay to ~ 1 second or less.
+
+Useful for simplying scripting or generically extending tools that don't support HBase High Availability directly
+
+By default checks the same --port on all servers. Hosts may have optional : suffixes added to individually
+override each one.
+
+Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print
+NO_AVAILABLE_SERVER.
+
+Tested on Kubernetes 1.13
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import os
+import sys
+import traceback
+#from random import shuffle
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from find_active_server import FindActiveServer
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.7.1'
+
+
+class FindActiveKubernetesAPI(FindActiveServer):
+
+ def __init__(self):
+ # Python 2.x
+ super(FindActiveKubernetesAPI, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.default_port = 8001
+ self.protocol = 'http'
+ # or just /healthz
+ self.url_path = '/healthz/ping'
+ self.regex = '^ok$'
+ self.default_num_threads = 2
+
+ def add_options(self):
+ self.add_hostoption(name='Kubernetes', default_port=self.default_port)
+ self.add_ssl_opt()
+ self.add_common_opts()
+
+ def process_options(self):
+ self.validate_common_opts()
+
+
+if __name__ == '__main__':
+ FindActiveKubernetesAPI().main()
diff --git a/find_active_oozie.py b/find_active_oozie.py
index 491afd818..156bb51a9 100755
--- a/find_active_oozie.py
+++ b/find_active_oozie.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -32,7 +32,7 @@
Exits with return code 1 and NO_AVAILABLE_SERVER if none of the namenodes are active, --quiet mode will not print
NO_AVAILABLE_SERVER.
-Tested on Oozie
+Tested on Oozie 4.2.0 on Hortonworks HDP 2.6.0
"""
diff --git a/find_active_presto_coordinator.py b/find_active_presto_coordinator.py
index 13c8f6546..9b0050a47 100755
--- a/find_active_presto_coordinator.py
+++ b/find_active_presto_coordinator.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_server.py b/find_active_server.py
index fe94f98fd..30810a448 100755
--- a/find_active_server.py
+++ b/find_active_server.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-29 15:00:36 +0100 (Thu, 29 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
r"""
@@ -96,7 +96,7 @@
See also Advanced HAProxy configurations (part of the Advanced Nagios Plugins Collection) at:
- https://github.com/harisekhon/haproxy-configs
+ https://github.com/HariSekhon/HAProxy-configs
"""
@@ -112,14 +112,20 @@
import subprocess
import sys
#from threading import Thread
-from multiprocessing.pool import ThreadPool, cpu_count
+from multiprocessing.pool import ThreadPool
+from multiprocessing import cpu_count
# prefer blocking semantics of que.get() rather than handling deque.popleft() => 'IndexError: pop from an empty deque'
#from collections import deque
-import Queue
import traceback
from random import shuffle
+# Python 2 Queue vs Python 3 queue module :-/
+if sys.version[0] == '2':
+ import Queue as queue # pylint: disable=import-error
+else:
+ import queue as queue # pylint: disable=import-error
try:
- import requests
+ # false positive from pylint, queue is imported first
+ import requests # pylint: disable=wrong-import-order
except ImportError:
print(traceback.format_exc(), end='')
sys.exit(4)
@@ -137,7 +143,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.8.2'
+__version__ = '0.8.6'
class FindActiveServer(CLI):
@@ -156,7 +162,7 @@ def __init__(self):
self.request_timeout = None
self.default_num_threads = min(cpu_count() * 4, 100)
self.num_threads = None
- self.queue = Queue.Queue()
+ self.queue = queue.Queue()
self.pool = None
def add_options(self):
@@ -333,7 +339,7 @@ def port_override(self, host):
def finish(self, host, port=None):
print(host, end='')
if port is not None and port != self.port:
- print(':{0}'.format(port))
+ print(':{0}'.format(port), end='')
print()
sys.exit(0)
@@ -419,7 +425,8 @@ def check_http(self, host, port, url_path=''):
log.info('%s - checking regex against content', url)
# if this ends up not being processed properly and remains a string instead
# of the expected compiled regex, then .search() will hang
- assert not isStr(self.regex)
+ if isStr(self.regex):
+ die('string found instead of expected compiled regex!')
if self.regex.search(req.content):
log.info('%s - regex matched http output', url)
else:
diff --git a/find_active_solrcloud.py b/find_active_solrcloud.py
index b6683fafb..659eceb59 100755
--- a/find_active_solrcloud.py
+++ b/find_active_solrcloud.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_duplicate_files.py b/find_duplicate_files.py
index 9b2c50d1e..4c452b03f 100755
--- a/find_duplicate_files.py
+++ b/find_duplicate_files.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-08-14 09:50:03 +0100 (Sun, 14 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -33,12 +33,17 @@
4. regex capture matching portion - specify a regex to match against the filenames with capture (brackets) and the
captured portion will be compared among files. If no capture brackets are detected
then will treat the entire regex as the capture.
- Regex is case insensitive by default
+ Regex is case insensitive by default and applies only to the file's basename
+
+Exits with exit code 4 if duplicates are found
Can restrict methods of finding duplicates to any combination of --name / --size / --checksum (checksum implies size as
an efficiency shortcut) / --regex. If none are specified then will try name, size + checksum. If specifying any one of
these options then the others will not run unless also explicitly specified.
+If you want to find files that are probably the same by byte count but may not have the same checksum due to minor
+corruption, such as large media files, then specify --size but do not specify --checksum which supercedes it
+
Caveats:
- The limitation of the checksum approach is that it can't determine files as duplicates if there is any
@@ -47,10 +52,12 @@
- By default this program will short-circuit to stop processing a file as soon as it is determined to be a duplicate
file via one of the above methods in that order for efficiency. This means that if 2 files have duplicate names,
and a third has a different name but the same checksum as the second one, the second one's size + checksum will not have
-been recorded stored and so the third duplicate will not be detected. However, if you removed one duplicate the next
-run of this program would find the other duplicate via the other dimension of checking. Given it's a rare condition
-it's probably not worth the extra overhead in everyday use but this behaviour can be overridden by specifying the
---no-short-circuit option too run every check on every file. Be aware this will slow down the process.
+been checked and so a third duplicate with a different name will not be detected by size / checksum. In most cases this
+is a good thing to finish quicker and avoid unnecessary checksumming which is computationally expensive and time
+consuming for large files. If you remove one duplicate then the next run of this program would find the other
+duplicate via the additional checks of size and checksumming. Given it's a rare condition it's probably not worth the
+extra overhead in everyday use but this behaviour can be overridden by specifying the --no-short-circuit option to run
+every check on every file. Be aware this will slow down the process.
To see progress of which files are matching size, backtracking to hash them for comparison etc
use --verbose twice or -vv. To see which files are being checked use triple verbose mode -vvv
@@ -81,7 +88,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.5.3'
+__version__ = '0.6.2'
class FindDuplicateFiles(CLI):
@@ -165,7 +172,7 @@ def process_args(self):
log_option('compare by name', self.compare_by_name)
log_option('compare by size', self.compare_by_size)
log_option('compare by checksum', self.compare_by_checksum)
- log_option('compare by regex', True if self.regex else False)
+ log_option('compare by regex', bool(self.regex))
return args
@staticmethod
@@ -210,36 +217,36 @@ def run(self):
for filepath in sorted(self.dup_filepaths):
print(filepath)
sys.exit(4)
- print('Duplicates detected!\n')
+ print('# Duplicates detected!')
if self.dups_by_name:
- print('Duplicates by name:\n')
+ print('\n# Duplicates by name:\n')
for basename in self.dups_by_name:
- print("--\nbasename '{0}':".format(basename))
+ print("# --\n# basename '{0}':".format(basename))
for filepath in sorted(self.dups_by_name[basename]):
print(filepath)
if self.dups_by_size:
- print('Duplicates by size:\n')
+ print('\n# Duplicates by size:\n')
for size in self.dups_by_size:
- print("--\nsize '{0}' bytes:".format(size))
+ print("# --\n# size '{0}' bytes:".format(size))
for filepath in sorted(self.dups_by_size[size]):
print(filepath)
if self.dups_by_hash:
- print('Duplicates by checksum:\n')
+ print('\n# Duplicates by checksum:\n')
for checksum in self.dups_by_hash:
- print("--\nchecksum '{0}':".format(checksum))
+ print("# --\n# checksum '{0}':".format(checksum))
for filepath in sorted(self.dups_by_hash[checksum]):
print(filepath)
if self.dups_by_regex:
- print('Duplicates by regex match ({0}):\n'.format(self.regex))
+ print('\n# Duplicates by regex match ({0}):\n'.format(self.regex))
for matching_portion in self.dups_by_regex:
- print("--\nregex matching portion '{0}':".format(matching_portion))
+ print("# --\n# regex matching portion '{0}':".format(matching_portion))
for filepath in sorted(self.dups_by_regex[matching_portion]):
print(filepath)
sys.exit(4)
elif self.failed:
sys.exit(2)
else:
- print('No Duplicates Found')
+ print('# No Duplicates Found')
sys.exit(0)
# def check_path(self, path):
@@ -270,7 +277,7 @@ def check_path(self, path):
# os.walk returns nothing if path is a file, and must store file names, sizes, checksums and regex captures
# even for standalone file args
if os.path.isfile(path):
- self.check_file(path)
+ self.is_file_dup(path)
elif os.path.isdir(path):
# returns generator
# root is the dir, dirs and files are child basenames
@@ -297,6 +304,7 @@ def check_path(self, path):
def is_file_dup(self, filepath):
log.debug("checking file path '%s'", filepath)
+ # pylint: disable=no-else-return
if os.path.islink(filepath):
log.debug("ignoring symlink '%s'", filepath)
return False
@@ -340,8 +348,7 @@ def is_file_dup_by_name(self, filepath):
self.dups_by_name[basename].add(self.files[basename])
self.dups_by_name[basename].add(filepath)
return True
- else:
- self.files[basename] = filepath
+ self.files[basename] = filepath
return False
def is_file_dup_by_size(self, filepath):
@@ -397,20 +404,21 @@ def is_file_dup_by_hash(self, filepath):
return False
def is_file_dup_by_regex(self, filepath):
- match = re.search(self.regex, filepath)
+ #match = re.search(self.regex, filepath)
+ basename = os.path.basename(filepath)
+ match = re.search(self.regex, basename)
if match:
log.debug("regex matched file '%s'", filepath)
if match.groups():
capture = match.group(1)
- if capture in self.regex_captures:
- self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set())
- self.dups_by_regex[capture].add(self.regex_captures[capture])
- self.dups_by_regex[capture].add(filepath)
- return True
- else:
- self.regex_captures[capture] = filepath
else:
- log.error('no capture detected! Did you forget to specify the (brackets) to capture in the regex?')
+ capture = match.group(0)
+ if capture in self.regex_captures:
+ self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set())
+ self.dups_by_regex[capture].add(self.regex_captures[capture])
+ self.dups_by_regex[capture].add(filepath)
+ return True
+ self.regex_captures[capture] = filepath
return False
diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py
new file mode 100755
index 000000000..f982cbddd
--- /dev/null
+++ b/find_missing_files_in_sequence.py
@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-07-31 11:03:17 +0100 (Fri, 31 Jul 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Finds missing files by numeric sequence, assuming a uniformly numbered file naming convention across files
+
+Files / directories are given as arguments or via standard input
+
+Directories are recursed and their files examined for missing numbers before each one
+
+Only supply files / directories that should be sharing a contiguously numbered file naming convention in each
+single run of this tool
+
+Accounts for zero padding in numbered files
+
+Caveats:
+
+- This is more complicated than you'd first think as there are so many file naming variations that no code could ever
+ be universally bulletproof and will likely require advanced regex tuning to match your use case and naming convention
+
+- Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be.
+ If you are looking for missing MP3 files, then you might be able to check the mp3 tag metadata using programs like
+ 'mediainfo' to get the total number of tracks and see if the files go that high
+
+- Returns globs by default instead of explicit missing filenames since suffixes can vary after numbers. If you have a
+ simple enough use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print
+ the missing files more explicitly, but in the general case you cannot account for suffix naming that isn't consistent,
+ such as chapters of audiobooks eg.
+
+ 'blah 01 - chapter about X.mp3'
+ 'blah 02 - chapter about Y.mp3'
+
+ so in the general case you cannot always infer suffixes, hence why it is left as globs. If you are sure that the
+ suffixes don't change then you can specify --fixed-suffix and it will infer each file's suffix as the basis for any
+ numerically missing files in the sequence, but if used where this is not the case, it'll generate a lot of false
+ positives that the default globbing mode would have handled
+
+- Doesn't currently find entire missing CD / disks in the naming format, but you should be able to see those cases
+ easily by eye
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import glob
+#import logging
+import os
+import re
+import sys
+import traceback
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ from harisekhon.utils import log, log_option, validate_regex, isInt, UnknownError
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3.2'
+
+
+# pylint: disable=too-many-instance-attributes
+class FindMissingFiles(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(FindMissingFiles, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.paths = []
+ self.regex_default = r'(? 1:
+ file_number = self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix)
+ if self.missing_files:
+ print('\n'.join(reversed(self.missing_files)))
+ self.missing_files = []
+
+ def determine_missing_file_backfill(self, file_prefix, file_number, padding, file_suffix):
+ file_number -= 1
+ if self.fixed_suffix:
+ explicit_last_filename = '{}{:0>%(padding)s}{}' % {'padding': padding}
+ explicit_last_filename = explicit_last_filename.format(file_prefix, file_number, file_suffix)
+ if not os.path.isfile(explicit_last_filename):
+ self.missing_files.append(explicit_last_filename)
+ else:
+ file_number = -1
+ else:
+ expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals()
+ expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number)
+ if not glob.glob(expected_last_filename_glob):
+ self.missing_files.append(expected_last_filename_glob)
+ else:
+ file_number = -1
+ return file_number
+
+
+if __name__ == '__main__':
+ FindMissingFiles().main()
diff --git a/gcp_cloud_function_ifconfig/.gcloudignore b/gcp_cloud_function_ifconfig/.gcloudignore
new file mode 100644
index 000000000..42de3fd00
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/.gcloudignore
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020)
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore
+
+.git
+.gcloudignore
+deploy.sh
+test/
+tests/
diff --git a/gcp_cloud_function_ifconfig/Makefile b/gcp_cloud_function_ifconfig/Makefile
new file mode 100644
index 000000000..656b78879
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/Makefile
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021)
+#
+# vim:ts=4:sts=4:sw=4:noet
+#
+# https://github.com/HariSekhon/Kubernetes-templates
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+SHELL = /usr/bin/env bash
+
+.PHONY: default
+default: deploy
+ @:
+
+.PHONY: deploy
+deploy:
+ @./deploy.sh
diff --git a/gcp_cloud_function_ifconfig/README.md b/gcp_cloud_function_ifconfig/README.md
new file mode 100644
index 000000000..0485fcf08
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/README.md
@@ -0,0 +1,45 @@
+Google Cloud Function - ifconfig
+=====================
+
+Queries http://ifconfig.co from GCF to check the routing and external IP being used eg. for comparison with Cloudflare / Firewall rules
+
+- `main.py` - the code
+- `requirements.txt` - the pip modules to bootstrap
+- `deploy.sh` - upload the code and deps
+
+Response is HTTP status code and message, then the raw JSON results
+
+```
+200 OK
+
+{
+ "ip": "1.2.3.4",
+ "ip_decimal": 1234567890,
+ "country": "United States",
+ "country_iso": "US",
+ "country_eu": false,
+ "latitude": 37.751,
+ "longitude": -97.822,
+ "time_zone": "America/Chicago",
+ "asn": "AS15169",
+ "asn_org": "GOOGLE",
+ "hostname": "ipv6.gae.googleusercontent.com",
+ "user_agent": {
+ "product": "python-requests",
+ "version": "2.24.0",
+ "raw_value": "python-requests/2.24.0"
+ }
+}
+```
+
+For IPv6 the format will be more like:
+```
+ "ip": "1234:5678:9012:34::a",
+ "ip_decimal": 12345678901234567890123456789012345678,
+```
+
+Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches:
+
+```
+./deploy.sh
+```
diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh
new file mode 100755
index 000000000..39aeba85e
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/deploy.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+cd "$srcdir"
+
+name="ifconfig"
+
+# Cloud Functions not available in all regions yet:
+#
+# https://cloud.google.com/functions/docs/locations
+#
+# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment
+region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)"
+region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}"
+
+# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com
+# for serverless VPC access to resources using their Private IPs
+# since we're only accessing the SQL Admin API we don't need this
+#vpc_connector="my-vpc-connector"
+
+opts=()
+if [ -n "${vpc_connector:-}" ]; then
+ # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules
+ opts+=(--vpc-connector "$vpc_connector" --egress-settings all)
+fi
+
+set -x
+gcloud functions deploy "$name" --trigger-http \
+ --security-level=secure-always \
+ --runtime python39 \
+ --entry-point main \
+ --memory 128MB \
+ --region "$region" \
+ --timeout 60 \
+ "${opts[@]}" \
+ --quiet # don't prompt to --allow-unauthenticated
+ #--max-instances 1 \
diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py
new file mode 100755
index 000000000..c9e50b0e7
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/main.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access
+routing via specified VPC Network to using default NAT Gateway
+
+Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules
+
+The HTTP request is irrelevant, just pass an empty JSON document '{}', although we could pass the website to query
+in a field in which case this would just act as a proxy.
+
+See Also: gcp_cloud_function_proxy/main.py
+
+
+Tested on GCP Cloud Functions with Python 3.9
+
+"""
+
+# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python
+
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+
+import requests
+
+def main(_):
+ """Responds to any HTTP request.
+ Args:
+ request (flask.Request): HTTP request object.
+ Returns:
+ The response text or any set of values that can be turned into a
+ Response object using
+ `make_response `.
+ """
+ req = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing
+ status_code = req.status_code
+ status_message = req.reason
+ content = req.text
+ return "{} {}\n\n{}".format(status_code, status_message, content)
diff --git a/gcp_cloud_function_ifconfig/requirements.txt b/gcp_cloud_function_ifconfig/requirements.txt
new file mode 100644
index 000000000..775d29ac6
--- /dev/null
+++ b/gcp_cloud_function_ifconfig/requirements.txt
@@ -0,0 +1,2 @@
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+requests==2.24.0
diff --git a/gcp_cloud_function_proxy/.gcloudignore b/gcp_cloud_function_proxy/.gcloudignore
new file mode 100644
index 000000000..42de3fd00
--- /dev/null
+++ b/gcp_cloud_function_proxy/.gcloudignore
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020)
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore
+
+.git
+.gcloudignore
+deploy.sh
+test/
+tests/
diff --git a/gcp_cloud_function_proxy/Makefile b/gcp_cloud_function_proxy/Makefile
new file mode 100644
index 000000000..656b78879
--- /dev/null
+++ b/gcp_cloud_function_proxy/Makefile
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021)
+#
+# vim:ts=4:sts=4:sw=4:noet
+#
+# https://github.com/HariSekhon/Kubernetes-templates
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+SHELL = /usr/bin/env bash
+
+.PHONY: default
+default: deploy
+ @:
+
+.PHONY: deploy
+deploy:
+ @./deploy.sh
diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md
new file mode 100644
index 000000000..1a4c2465a
--- /dev/null
+++ b/gcp_cloud_function_proxy/README.md
@@ -0,0 +1,27 @@
+Google Cloud Function - Proxy
+=====================
+
+Queries a given URL from GCF to check connectivity eg. for testing with Cloudflare / Firewall rules
+
+Query content:
+```
+{"url": "http://ifconfig.co/json"}
+```
+
+Response is HTTP status code and message, blank line and then the content:
+
+```
+200 OK
+
+
+```
+
+- `main.py` - the code
+- `requirements.txt` - the pip modules to bootstrap
+- `deploy.sh` - upload the code and deps
+
+Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches:
+
+```
+./deploy.sh
+```
diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh
new file mode 100755
index 000000000..d8e04e9b5
--- /dev/null
+++ b/gcp_cloud_function_proxy/deploy.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+cd "$srcdir"
+
+name="proxy"
+
+# Cloud Functions not available in all regions yet:
+#
+# https://cloud.google.com/functions/docs/locations
+#
+# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment
+region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)"
+region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}"
+
+# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com
+# for serverless VPC access to resources using their Private IPs
+# since we're only accessing the SQL Admin API we don't need this
+#vpc_connector="my-vpc-connector"
+
+opts=()
+if [ -n "${vpc_connector:-}" ]; then
+ # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules
+ opts+=(--vpc-connector "$vpc_connector" --egress-settings all)
+fi
+
+set -x
+gcloud functions deploy "$name" --trigger-http \
+ --security-level=secure-always \
+ --runtime python39 \
+ --entry-point main \
+ --memory 128MB \
+ --region "$region" \
+ --timeout 60 \
+ "${opts[@]}" \
+ --quiet # don't prompt to --allow-unauthenticated
+ #--max-instances 1
diff --git a/gcp_cloud_function_proxy/main.py b/gcp_cloud_function_proxy/main.py
new file mode 100755
index 000000000..8680d455a
--- /dev/null
+++ b/gcp_cloud_function_proxy/main.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access
+routing via specified VPC Network to using default NAT Gateway
+
+Example usage:
+
+Check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules
+
+Test request examples:
+
+ { "url": "http://ifconfig.co/json" }
+
+defaults to http:// if not specified:
+
+ { "url": "ifconfig.co/json" }
+
+
+Tested on GCP Cloud Functions with Python 3.9
+
+"""
+
+# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python
+
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+
+import json
+import requests
+
+def main(request):
+ """Responds to any HTTP request.
+ Args:
+ request (flask.Request): HTTP request object.
+ Returns:
+ The response text or any set of values that can be turned into a
+ Response object using
+ `make_response `.
+ """
+ data = json.loads(request.data)
+ url = data['url']
+ if '://' not in url:
+ url = 'http://' + url
+ req = requests.get(url)
+ status_code = req.status_code
+ status_message = req.reason
+ content = req.text
+ return "{} {}\n\n{}".format(status_code, status_message, content)
diff --git a/gcp_cloud_function_proxy/requirements.txt b/gcp_cloud_function_proxy/requirements.txt
new file mode 100644
index 000000000..775d29ac6
--- /dev/null
+++ b/gcp_cloud_function_proxy/requirements.txt
@@ -0,0 +1,2 @@
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+requests==2.24.0
diff --git a/gcp_cloud_function_sql_export/.gcloudignore b/gcp_cloud_function_sql_export/.gcloudignore
new file mode 100644
index 000000000..42de3fd00
--- /dev/null
+++ b/gcp_cloud_function_sql_export/.gcloudignore
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020)
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore
+
+.git
+.gcloudignore
+deploy.sh
+test/
+tests/
diff --git a/gcp_cloud_function_sql_export/Makefile b/gcp_cloud_function_sql_export/Makefile
new file mode 100644
index 000000000..656b78879
--- /dev/null
+++ b/gcp_cloud_function_sql_export/Makefile
@@ -0,0 +1,22 @@
+#
+# Author: Hari Sekhon
+# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021)
+#
+# vim:ts=4:sts=4:sw=4:noet
+#
+# https://github.com/HariSekhon/Kubernetes-templates
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+SHELL = /usr/bin/env bash
+
+.PHONY: default
+default: deploy
+ @:
+
+.PHONY: deploy
+deploy:
+ @./deploy.sh
diff --git a/gcp_cloud_function_sql_export/README.md b/gcp_cloud_function_sql_export/README.md
new file mode 100644
index 000000000..db3ff61ad
--- /dev/null
+++ b/gcp_cloud_function_sql_export/README.md
@@ -0,0 +1,52 @@
+Google Cloud Function - SQL Backup Exporter to GCS
+=====================
+
+Triggers GCP [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage).
+
+Solution documentation:
+
+https://cloud.google.com/solutions/scheduling-cloud-sql-database-exports-using-cloud-scheduler
+
+- `main.py` - the code
+- `requirements.txt` - the pip modules to bootstrap
+- `deploy.sh` - upload the code and deps
+
+Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches:
+
+```
+./deploy.sh
+```
+
+### Solution Dependencies
+
+- a [Cloud PubSub](https://cloud.google.com/pubsub) topic
+- [Cloud Scheduler](https://cloud.google.com/scheduler) jobs to trigger backups
+ - see `gcp_cloud_schedule_sql_exports.sh` in [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo
+- a service account with permissions to access [Cloud SQL](https://cloud.google.com/sql)
+ - see `gcp_sql_create_readonly_service_account.sh` in [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo
+- each [Cloud SQL](https://cloud.google.com/sql) instance to be backed up requires objectCreator permissions to the [GCS](https://cloud.google.com/storage) bucket
+ - see `gcp_sql_grant_instances_gcs_object_creator.sh` in [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo
+
+### Serverless Framework
+
+Instead of `deploy.sh` you can alternatively use the [Serverless](https://www.serverless.com/) framework for which a `serverless.yml` config is provided:
+
+```
+serverless deploy
+```
+
+If this is your first time using Serverless then you'll need to install the GCP plugin:
+
+```
+serverless plugin install --name serverless-google-cloudfunctions
+```
+
+The `serverless.yml` config expects to find `$GOOGLE_PROJECT_ID` and `$GOOGLE_REGION` environment variables.
+
+Serverless requires additional permissions for the service account: Deployment Manager Editor and Storage Admin to create deployments and staging buckets.
+
+You can also build a serverless artifact to `.serverless/` without deploying it (generates Google [Deployment Manager](https://cloud.google.com/deployment-manager) templates and a zip file - useful to check what would be uploaded / ignored):
+
+```
+serverless package
+```
diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh
new file mode 100755
index 000000000..c5c3c3249
--- /dev/null
+++ b/gcp_cloud_function_sql_export/deploy.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-10-16 10:12:26 +0100 (Fri, 16 Oct 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+cd "$srcdir"
+
+# needed to define the $service_account further down
+project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config list --format="value(core.project)")}"
+
+# Cloud Functions not available in all regions yet:
+#
+# https://cloud.google.com/functions/docs/locations
+#
+# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment
+region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)"
+region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}"
+
+name="cloud-sql-backups"
+topic="cloud-sql-backups"
+service_account="cloud-function-sql-backup@$project.iam.gserviceaccount.com"
+# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com
+# for serverless VPC access to resources using their Private IPs
+# since we're only accessing the SQL Admin API we don't need this
+#vpc_connector="cloud-sql-backups"
+
+gcloud functions deploy "$name" --trigger-topic "$topic" \
+ --runtime python37 \
+ --entry-point main \
+ --service-account "$service_account" \
+ --region "$region" \
+ --memory 128MB \
+ --timeout 60
+ # may want multiple concurrent calls to different SQL instances at the same time
+ #
+ # also doesn't prevent:
+ #
+ # "Operation failed because another operation was already in progress."
+ #
+ #--max-instances 1 # this isn't good enough because it sets off an async API call, so successive calls can fail if called before that SQL Admin API export has finished
+
+ # --vpc-connector "$vpc_connector"
diff --git a/gcp_cloud_function_sql_export/main.py b/gcp_cloud_function_sql_export/main.py
new file mode 100755
index 000000000..6e3765667
--- /dev/null
+++ b/gcp_cloud_function_sql_export/main.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-10-14 15:29:38 +0100 (Wed, 14 Oct 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+GCP Cloud Function to export a given Cloud SQL database to GCS via PubSub notifications from Cloud Scheduler
+
+Solution Documentation:
+
+ https://cloud.google.com/solutions/scheduling-cloud-sql-database-exports-using-cloud-scheduler
+
+GCP Cloud PubSub should be sent payloads like this by Cloud Scheduler (replacing the env vars with your literals):
+
+{
+ "project": "${GOOGLE_PROJECT_ID}",
+ "instance": "${SQL_INSTANCE_HOST}",
+ "database": "${DATABASE}",
+ "bucket": "${BUCKET_NAME}"
+}
+
+Tested on GCP Cloud Functions with Python 3.7
+
+"""
+
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+
+# Code below is based on solution sample code from the link above
+
+#import os
+import base64
+import logging
+import json
+
+from datetime import datetime
+from httplib2 import Http
+
+from googleapiclient import discovery
+from googleapiclient.errors import HttpError
+from oauth2client.client import GoogleCredentials
+
+
+# pylint: disable=unused-argument
+def main(event, context):
+# if os.getenv("DEBUG"):
+# # debug level logs don't appear in function details logs tab even with
+# # DEBUG=1 runtime env var set and logs severity filter set to >= DEBUG
+# #logging.debug('event: %s', event)
+# #logging.debug('context: %s', context)
+# logging.info('event: %s', event)
+# logging.info('context: %s', context)
+ data = json.loads(base64.b64decode(event['data']).decode('utf-8'))
+ credentials = GoogleCredentials.get_application_default()
+
+ service = discovery.build('sqladmin', 'v1beta4', http=credentials.authorize(Http()), cache_discovery=False)
+
+ project = data['project']
+ bucket = data['bucket']
+ bucket = bucket.lstrip('gs://')
+ instance = data['instance']
+ database = data['database']
+ timestamp = datetime.now().strftime("%Y-%m-%d_%H%M")
+
+ # .gz extension so it is auto-compressed, saving storage space + billing
+ backup_uri = "gs://{bucket}/backups/sql/{instance}--{database}--{timestamp}.sql.gz".format(
+ bucket=bucket,
+ instance=instance,
+ database=database,
+ timestamp=timestamp)
+
+ instances_export_request_body = {
+ "exportContext": {
+ "kind": "sql#exportContext",
+ "fileType": "SQL",
+ "uri": backup_uri,
+ "databases": [
+ database
+ ]
+ }
+ }
+
+ try:
+ logging.info("Requesting project '%s' database instance '%s' runs a backup export to bucket '%s' path '%s'",
+ project,
+ instance,
+ bucket,
+ backup_uri)
+ request = service.instances().export(
+ project=project,
+ instance=instance,
+ body=instances_export_request_body
+ )
+ response = request.execute()
+ except HttpError as err:
+ logging.error("Backup FAILED. Reason: %s", err)
+ else:
+ logging.info("Backup Task triggered: %s", response)
diff --git a/gcp_cloud_function_sql_export/requirements.txt b/gcp_cloud_function_sql_export/requirements.txt
new file mode 100644
index 000000000..2f7349f4d
--- /dev/null
+++ b/gcp_cloud_function_sql_export/requirements.txt
@@ -0,0 +1,3 @@
+# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python
+google-api-python-client==1.12.5
+oauth2client==4.1.3
diff --git a/gcp_cloud_function_sql_export/serverless.yml b/gcp_cloud_function_sql_export/serverless.yml
new file mode 100644
index 000000000..a73986efb
--- /dev/null
+++ b/gcp_cloud_function_sql_export/serverless.yml
@@ -0,0 +1,137 @@
+# vim:ts=2:sts=2:sw=2:et
+#
+# Author: Hari Sekhon
+# Date: 2020-10-21 11:19:06 +0100 (Wed, 21 Oct 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e r v e r l e s s
+# ============================================================================ #
+
+# Deploys the Google Cloud Function with a name --
+#
+# eg. cloud-sql-backups-production-main
+
+# Requires the same permissions as the simpler adjacent deploy.sh script but also:
+#
+# Deployment Manager Editor
+# Storage Admin (Storage Object Admin is not enough as it needs to create staging buckets and you'll get weird errors otherwise)
+#
+# for the credential file's service account which is also used to run the Cloud Function
+
+# Check generated config after environment variable interpolation:
+#
+# cd "$(dirname $0)" && serverless print
+#
+# My advanced vimrc has a hotkey for this:
+#
+# https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.vimrc
+
+frameworkVersion: '2'
+plugins:
+ - serverless-google-cloudfunctions
+
+# to prevent:
+#
+# Serverless: Configuration warning: Unrecognized provider 'google'
+# Serverless:
+# Serverless: You're relying on provider plugin which doesn't provide a validation schema for its config.
+configValidationMode: off
+# after the GCP plugin matures, switch to
+#configValidationMode: error
+
+service: cloud-sql-backups
+provider:
+ name: google
+ stage: production
+ runtime: python37
+ region: ${env:GOOGLE_REGION, "europe-west1"}
+ project: ${env:GOOGLE_PROJECT_ID}
+
+ # https://serverless.com/framework/docs/providers/google/guide/credentials/
+ #
+ # path to credentials file needs to be absolute
+ #
+ # download a credentials file for the service account:
+ #
+ # https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-gcloud
+ #
+ # mkdir -pv ~/.gcloud && gcloud iam service-accounts keys create ~/.gcloud/cloud-function-sql-backup-keyfile.json --iam-account "cloud-function-sql-backup@$(gcloud config list --format="get(core.project)").iam.gserviceaccount.com"
+ #
+ credentials: ~/.gcloud/cloud-function-sql-backup-keyfile.json
+ #
+ # or use personal creds - see $GOOGLE_APPLICATION_CREDENTIALS or generate:
+ #
+ # gcloud auth application-default login # [ --client-id-file=~/.gcloud/keyfile.json ]
+ #
+ #credentials: ~/.config/gcloud/application_default_credentials.json
+ #credentials: ${env:HOME}/.config/gcloud/application_default_credentials.json
+ #credentials: /Users/harisekhon/.config/gcloud/application_default_credentials.json
+
+# https://www.serverless.com/framework/docs/providers/google/guide/packaging/
+package:
+ exclude:
+ ## needs more granular excluding in production as only the serverless provider npm
+ ## package should be excluded (and not the whole node_modules directory)
+ #- node_modules/**
+ #- deploy.sh
+ #- test/**
+ #- tests/**
+ #- .gitignore
+ #- .gcloudignore
+ #- .git/**
+ #- package.json
+ #- package-lock.json
+ #- README.md
+ # XXX: exclude all hidden files and directories
+ - .*
+ # XXX: more robust to only whitelist include as per .dockerignore best practice too (unfortunately not supported in .gcloudignore)
+ - ./**
+ include:
+ - main.py
+ - requirements.txt
+ # or specify your own zipfile and skip packaging:
+ #artifact: path/to/my-artifact.zip
+
+# https://www.serverless.com/framework/docs/providers/google/guide/functions/
+#
+# use an array of includes for bigger serverless deployments of multiple functions
+#functions:
+# - ${file(../foo-functions.yml)}
+# - ${file(../bar-functions.yml)}
+#
+functions:
+ #myfunc:
+ # handler: http
+ # events:
+ # - http: path
+ # NOTE: the following uses an "event" event (pubSub event in this case).
+ # Please create the corresponding resources in the Google Cloud
+ # before deploying this service through Serverless
+ main:
+ handler: main
+ memorySize: 128
+ timeout: 60s
+ events:
+ - event:
+ eventType: providers/cloud.pubsub/eventTypes/topic.publish
+ #resource: projects/*/topics/my-topic
+ resource: projects/${env:GOOGLE_PROJECT_ID}/topics/cloud-sql-backups
+# you can define resources, templates etc. the same way you would in a
+# Google Cloud deployment configuration
+resources:
+ resources:
+ # generates the bucket that the cloud function will send the backups to
+ # there is another bucket also created for uploading the package (sls-cloud-sql-backups-production-1603283101338), not to be confused with this one
+ - type: storage.v1.bucket
+ name: ${env:GOOGLE_PROJECT_ID}-sql-backups
+# imports:
+# - path: my_template.jinja
diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py
new file mode 100755
index 000000000..f58f86e98
--- /dev/null
+++ b/gcp_service_account_credential_keys.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-10-29 18:02:14 +0000 (Thu, 29 Oct 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Lists all service account credential keys in a given GCP project
+
+Excludes built-in system managed keys which are hidden in the Console UI anyway and are not actionable
+or in scope for a key policy audit.
+
+
+Output Format:
+
+
+
+
+You can supply a service account credentials file to authenticate with or just use ADC via:
+
+ gcloud auth application-default login
+
+
+See Also - similar scripts in the DevOps Bash tools repo:
+
+ https://github.com/HariSekhon/DevOps-Bash-tools/
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from datetime import datetime
+import json
+import os
+import sys
+import traceback
+from google.oauth2 import service_account
+import googleapiclient.discovery
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, log_option, validate_int
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1'
+
+
+class GcpServiceAccountCredentialKeys(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(GcpServiceAccountCredentialKeys, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.credentials_file = None
+ self.service = None
+ self.project = None
+ self.no_expiry = None
+ self.expired = None
+ self.expires_within_days = None
+
+ def add_options(self):
+ super(GcpServiceAccountCredentialKeys, self).add_options()
+ self.add_opt('-f', '--credentials-file', metavar='',
+ default=os.getenv('GOOGLE_CREDENTIALS', \
+ os.getenv('GOOGLE_APPLICATION_CREDENTIALS')),
+ help='Credentials file ($GOOGLE_CREDENTIALS, ' + \
+ '$GOOGLE_APPLICATION_CREDENTIALS)')
+ self.add_opt('-p', '--project-id', metavar='',
+ help='Google Cloud Project ID ($GOOGLE_PROJECT_ID, or inferred from credentials file')
+ self.add_opt('-n', '--no-expiry', action='store_true', help='List only non-expiring keys')
+ self.add_opt('-e', '--expired', action='store_true', help='List only expired keys')
+ self.add_opt('-d', '--expires-within-days', type=int, help='List only keys that will expire within N days')
+
+ def process_options(self):
+ super(GcpServiceAccountCredentialKeys, self).process_options()
+ self.no_args()
+ project_id = self.get_opt('project_id')
+ credsfile = self.get_opt('credentials_file')
+ self.no_expiry = self.get_opt('no_expiry')
+ self.expired = self.get_opt('expired')
+ self.expires_within_days = self.get_opt('expires_within_days')
+ #if not credsfile:
+ # self.usage('no --credentials-file given and ' + \
+ # 'GOOGLE_CREDENTIALS / GOOGLE_APPLICATION_CREDENTIALS environment variables not populated')
+ if credsfile:
+ if not os.path.exists(credsfile):
+ self.usage('credentials file not found: {}'.format(credsfile))
+ self.credentials_file = credsfile
+ log_option('credentials file', self.credentials_file)
+ if not project_id:
+ if credsfile:
+ json_data = json.loads(open(credsfile).read())
+ project_id = json_data['project_id']
+ else:
+ self.usage('--project-id not specified and no credentials file given from which to infer')
+ self.project = project_id
+ log_option('project', self.project)
+ if self.expires_within_days is not None:
+ validate_int(self.expires_within_days, 'expires within days', 0)
+ if self.no_expiry:
+ self.usage('--expires-within-days and --no-expiry are mutually exclusive')
+ if self.expired:
+ self.usage('--expires-within-days and --expired are mutually exclusive')
+ if self.no_expiry and self.expired:
+ self.usage('--expired and --no-expiry are mutually exclusive')
+
+ def run(self):
+ # defaults to looking for $GOOGLE_APPLICATION_CREDENTIALS or using Application Default Credentials
+ # from 'gcloud auth application-default login' => ~/.config/gcloud/application_default_credentials.json
+ credentials = None
+ if self.credentials_file:
+ log.debug('loading credentials')
+ credentials = service_account.Credentials.from_service_account_file(
+ filename=self.credentials_file,
+ scopes=['https://www.googleapis.com/auth/cloud-platform']
+ )
+
+ # cache_discovery=False avoids:
+ # ImportError: file_cache is unavailable when using oauth2client >= 4.0.0 or google-auth
+ self.service = googleapiclient.discovery.build('iam', 'v1', credentials=credentials, cache_discovery=False)
+
+ for service_account_email in self.get_service_accounts():
+ self.list_keys(service_account_email)
+
+ def get_service_accounts(self):
+ """ Returns a list of service account email addresses """
+
+ log.debug('getting service accounts')
+ service_accounts = self.service.projects()\
+ .serviceAccounts()\
+ .list(name='projects/{}'.format(self.project))\
+ .execute()
+ for account in service_accounts['accounts']:
+ yield account['email']
+
+ def list_keys(self, service_account_email):
+ log.debug("getting keys for service account '%s'", service_account_email)
+ keys = self.service.projects()\
+ .serviceAccounts()\
+ .keys()\
+ .list(name='projects/-/serviceAccounts/' + service_account_email).execute()
+
+ for key in keys['keys']:
+ if key['keyType'] == 'SYSTEM_MANAGED':
+ continue
+ _id = key['name'].split('/')[-1]
+ created_date = key['validAfterTime']
+ expiry_date = key['validBeforeTime']
+ created_datetime = datetime.strptime(created_date, "%Y-%m-%dT%H:%M:%SZ")
+ expiry_datetime = datetime.strptime(expiry_date, "%Y-%m-%dT%H:%M:%SZ")
+ age_timedelta = datetime.utcnow() - created_datetime
+ age_days = int(age_timedelta.total_seconds() / 86400)
+ expired = False
+ if expiry_date == '9999-12-31T23:59:59Z':
+ expires_in_days = 'NEVER'
+ else:
+ expires_in_timedelta = expiry_datetime - datetime.utcnow()
+ expires_in_days = int(expires_in_timedelta.total_seconds() / 86400)
+ if expires_in_days < 1:
+ expired = True
+ if self.no_expiry and (expires_in_days != 'NEVER'):
+ continue
+ if self.expired and expired:
+ continue
+ if self.expires_within_days is not None and \
+ (expires_in_days == 'NEVER' or expires_in_days > self.expires_within_days):
+ continue
+ print('{id} {created} {expires} {age:4d} {expires_in:5s} {expired} {service_account}'.format(
+ id=_id,
+ created=created_date,
+ expires=expiry_date,
+ age=age_days,
+ expires_in=expires_in_days,
+ expired=expired,
+ service_account=service_account_email
+ ))
+
+
+if __name__ == '__main__':
+ GcpServiceAccountCredentialKeys().main()
diff --git a/getent.py b/getent.py
index ce77aefcf..d8b701cd6 100755
--- a/getent.py
+++ b/getent.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-11-20 12:35:49 +0000 (Sun, 20 Nov 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/git_check_branches_upstream.py b/git_check_branches_upstream.py
index 0ab2e177b..27ec09865 100755
--- a/git_check_branches_upstream.py
+++ b/git_check_branches_upstream.py
@@ -1,24 +1,24 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-07-21 16:19:19 +0100 (Thu, 21 Jul 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
Tool to check Git branches have their upstream origin branch set consistently and auto-fix if necessary
-Mainly written for my https://github.com/harisekhon/Dockerfiles repo
+Mainly written for my https://github.com/HariSekhon/Dockerfiles repo
which has over 100 branches which get merged, pulled and pushed around
"""
diff --git a/hbase_compact_tables.py b/hbase_compact_tables.py
index 60f246328..7bfca40b2 100755
--- a/hbase_compact_tables.py
+++ b/hbase_compact_tables.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-04-27 20:49:23 +0100 (Wed, 27 Apr 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_flush_tables.py b/hbase_flush_tables.py
index 4da3bb492..71be25c17 100755
--- a/hbase_flush_tables.py
+++ b/hbase_flush_tables.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-12 13:44:24 +0200 (Mon, 12 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# NOTE: 'flush' is not supported in Happybase as it's not in the Thrift API,
@@ -27,6 +27,8 @@
The Thrift API doesn't support this action so it uses the HBase shell locally which must be in the $PATH
+There is also a shell script version of this in the adjacent DevOps-Perl-Tools repo
+
Tested on Hortonworks HDP 2.3 (HBase 1.1.2) and Apache HBase 1.0.3, 1.1.6, 1.2.1, 1.2.2, 1.3.1
"""
@@ -42,7 +44,7 @@
import sys
import traceback
import subprocess
-from subprocess import PIPE
+PIPE = subprocess.PIPE
libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
sys.path.append(libdir)
try:
diff --git a/hbase_generate_data.py b/hbase_generate_data.py
index 133fa42dc..3ed31a644 100755
--- a/hbase_generate_data.py
+++ b/hbase_generate_data.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-14 15:19:35 +0200 (Wed, 14 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_region_requests.py b/hbase_region_requests.py
index 68ec05500..d182431c8 100755
--- a/hbase_region_requests.py
+++ b/hbase_region_requests.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -50,7 +50,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.5.2'
+__version__ = '0.5.3'
class HBaseRegionsRequests(CLI):
@@ -88,6 +88,7 @@ def add_options(self):
self.add_opt('--reads', action='store_true', help='Show read requests (default shows read and write)')
self.add_opt('--writes', action='store_true', help='Show write requests (default shows read and write')
self.add_opt('--total', action='store_true', help='Show total requests (default shows read and write)')
+ self.add_opt('--skip-zeros', action='store_true', help="Don't output regions which have zero requests")
def process_args(self):
self.host_list = self.args
@@ -211,6 +212,7 @@ def process_bean(self, host, bean, uptime):
def print_stats(self, host):
stats = self.stats
show = self.show
+ skip_zeros = self.get_opt('skip_zeros')
tstamp = time.strftime('%F %T')
if not stats:
print("No table regions found for table '{}'. Did you specify the correct table name?".format(self.table))
@@ -221,6 +223,7 @@ def print_stats(self, host):
.format(tstamp, host, self.interval, plural(self.interval)))
self.first_iteration = 0
return
+ count = 0
for table in sorted(stats[host]):
for region in sorted(stats[host][table]):
table_region = region
@@ -229,10 +232,16 @@ def print_stats(self, host):
# maintain explicit order for humans
# rather than iterate keys of region which will some out in the wrong order
for metric in ('read', 'write', 'total'):
- if (not show) or metric in show:
- print('{:20s}\t{:20s}\t{:40s}\t{:10s}\t{:8.0f}'\
- .format(tstamp, host, table_region, metric, stats[host][table][region][metric]))
- print()
+ value = stats[host][table][region][metric]
+ if skip_zeros and int(value) == 0:
+ continue
+ if show and metric not in show:
+ continue
+ print('{:20s}\t{:20s}\t{:40s}\t{:10s}\t{:8.0f}'\
+ .format(tstamp, host, table_region, metric, value))
+ count += 1
+ if count:
+ print()
# some extra effort to make it look the same as HBase presents it as
#def encode_char(self, char):
diff --git a/hbase_regions_by_memstore_size.py b/hbase_regions_by_memstore_size.py
index 47165a7a9..295653a81 100755
--- a/hbase_regions_by_memstore_size.py
+++ b/hbase_regions_by_memstore_size.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regions_by_size.py b/hbase_regions_by_size.py
index 49f7419d8..c67184c8b 100755
--- a/hbase_regions_by_size.py
+++ b/hbase_regions_by_size.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regions_least_used.py b/hbase_regions_least_used.py
index 45727dd34..752be4ed5 100755
--- a/hbase_regions_least_used.py
+++ b/hbase_regions_least_used.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regionserver_requests.py b/hbase_regionserver_requests.py
index f61b40cbd..a8c96ebb9 100755
--- a/hbase_regionserver_requests.py
+++ b/hbase_regionserver_requests.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_scan_table_column_names.sh b/hbase_scan_table_column_names.sh
index 08e6ab5a1..026f7d5a8 100755
--- a/hbase_scan_table_column_names.sh
+++ b/hbase_scan_table_column_names.sh
@@ -4,20 +4,20 @@
# Author: Hari Sekhon
# Date: 2018-06-29 19:01:22 +0100 (Fri, 29 Jun 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
usage(){
- [ -n "${@:-}" ] && echo "$@"
+ [ -n "${*:-}" ] && echo "$@"
echo
echo "Fast and dirty script to scan an HBase table to determine all columns (column family + qualifiers)
@@ -45,7 +45,7 @@ if [ -z "${table:-}" ]; then
usage "ERROR: Table not defined, use --table switch"
fi
-if ! which hbase &>/dev/null; then
+if ! type -P hbase &>/dev/null; then
usage "ERROR: hbase command not found in \$PATH ($PATH)"
fi
diff --git a/hbase_show_table_region_ranges.py b/hbase_show_table_region_ranges.py
index 2a5647fb9..585e11cf9 100755
--- a/hbase_show_table_region_ranges.py
+++ b/hbase_show_table_region_ranges.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-05 13:57:37 +0100 (Wed, 05 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_table_region_row_distribution.py b/hbase_table_region_row_distribution.py
index 87b4d2e34..0dfe6e024 100755
--- a/hbase_table_region_row_distribution.py
+++ b/hbase_table_region_row_distribution.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_table_regions_by_regionserver.sh b/hbase_table_regions_by_regionserver.sh
index b19510d8f..8017f406a 100755
--- a/hbase_table_regions_by_regionserver.sh
+++ b/hbase_table_regions_by_regionserver.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-08-24 19:34:33 +0100 (Fri, 24 Aug 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -18,7 +18,6 @@ set -euo pipefail
regionservers=""
port="${HBASE_REGIONSERVER_PORT:-16030}"
-metrics="metrics"
usage(){
if [ -n "$*" ]; then
@@ -40,7 +39,7 @@ usage: ${0##*/} [options] regionserver1 regionserver2 regionserver3 ...
-p --port RegionServer HTTP JMX port
EOF
- exit 1
+ exit 3
}
table=".*"
@@ -66,7 +65,8 @@ fi
check_bin(){
local bin="$1"
- if ! which $bin &>/dev/null; then
+ # shellcheck disable=SC2230
+ if ! type -P "$bin" &>/dev/null; then
echo "'$bin' command not found in \$PATH ($PATH)"
exit 1
fi
@@ -79,6 +79,7 @@ if [ -z "${DEBUG:-}" ]; then
fi
for regionserver in $(tr ' ' '\n' <<< "$regionservers" | sort -u); do
echo "$regionserver:"
+ # shellcheck disable=SC2086
curl $curl_options "$regionserver:$port/jmx" |
grep "_table_.*${table}.*_region_" |
sed 's/.*_table_//; s/_region_/ /; s/_.*$//' |
diff --git a/hbase_table_regionserver_distribution.sh b/hbase_table_regionserver_distribution.sh
index e7f5c69cd..58699a877 100755
--- a/hbase_table_regionserver_distribution.sh
+++ b/hbase_table_regionserver_distribution.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-11 19:36:23 +0100 (Wed, 11 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Useful to see table region skew across regionservers
@@ -24,7 +24,6 @@
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
usage(){
echo "usage: $0 "
@@ -37,9 +36,9 @@ if [ $# -lt 1 ]; then
usage
fi
-for server in $@; do
+for server in "$@"; do
echo "$server:"
- curl -s $server:$port/jmx |
+ curl -s "$server:$port/jmx" |
grep 'Namespace_.*_metric_getCount"[[:space:]]*:[[:space:]]*' |
sed 's/.*_table_//; s/_region_.*//' |
sort |
diff --git a/hbase_table_row_key_distribution.py b/hbase_table_row_key_distribution.py
index 8784d5952..6059763d7 100755
--- a/hbase_table_row_key_distribution.py
+++ b/hbase_table_row_key_distribution.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-08 09:02:01 +0100 (Sat, 08 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_uncompacted_regions.sh b/hbase_uncompacted_regions.sh
index e4afcd96a..00a3302c5 100755
--- a/hbase_uncompacted_regions.sh
+++ b/hbase_uncompacted_regions.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-11 19:26:21 +0100 (Wed, 11 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Find regions that require major_compact after region migration to achieve data locality again
@@ -19,7 +19,6 @@
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
usage(){
echo "usage: $0 "
@@ -32,8 +31,8 @@ if [ $# -lt 1 ]; then
usage
fi
-for server in $@; do
- curl -s $server:$port/jmx |
+for server in "$@"; do
+ curl -s "$server:$port/jmx" |
grep 'compactionsCompletedCount"[[:space:]]*:[[:space:]]*0,$' |
sed 's/.*_table_//; s/_region_/ /; s/_metric_.*//' |
column -t
diff --git a/hadoop_hdfs_files_native_checksums.jy b/hdfs_files_native_checksums.jy
similarity index 94%
rename from hadoop_hdfs_files_native_checksums.jy
rename to hdfs_files_native_checksums.jy
index 29ed7e80e..9c558401a 100755
--- a/hadoop_hdfs_files_native_checksums.jy
+++ b/hdfs_files_native_checksums.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2013-06-20 18:21:02 +0100 (Thu, 20 Jun 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -15,7 +15,7 @@ from __future__ import print_function
from __future__ import unicode_literals
__author__ = 'Hari Sekhon'
-__version__ = '0.5.0'
+__version__ = '0.5.1'
# pylint: disable=wrong-import-position
import os
@@ -50,7 +50,7 @@ def usage(*msg):
if msg:
printerr("".join(msg))
die("""
-Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
================================================================================
%s - version %s
@@ -58,7 +58,7 @@ Hari Sekhon - https://github.com/harisekhon/devops-python-tools
Jython program to fetch the HDFS native checksums for given files / all files under given directories.
-Quick way of checking for duplicate files but not checking content itself. This is at least ~100x more efficient in terms of data transfer than 'hadoop fs -cat | md5sum'.
+Quick way of checking for duplicate files but not checking content itself. This is ~10x more efficient in terms of data transfer than 'hadoop fs -cat | md5sum'.
Caveat: files with differing block sizes will not match
Will be implemented in hadoop-9209 jira for versions 3.0.0, 0.23.7, 2.1.0-beta using a new command line param: hadoop fs -checksum
@@ -142,7 +142,7 @@ class HDFSChecksumReader(object):
printerr("Failed to get HDFS path object for file " + filename, 1)
continue
if not self.filesystem.exists(path):
- #raise IOError, "HDFS File not found: %s" % filename
+ #raise IOError("HDFS File not found: %s" % filename)
printerr("HDFS file/dir not found: %s" % filename, 1)
continue
self.print_checksum_recurse(filename, path)
@@ -161,7 +161,7 @@ class HDFSChecksumReader(object):
except:
printerr(get_jython_exception().split("\n")[0], 1)
else:
- raise IOError, ">>> %s is not a file or directory" % filename
+ raise IOError(">>> %s is not a file or directory" % filename)
def print_checksum(self, filename, path):
""" Prints the HDFS checksum for the given file + fs path object """
diff --git a/hadoop_hdfs_files_stats.jy b/hdfs_files_stats.jy
similarity index 93%
rename from hadoop_hdfs_files_stats.jy
rename to hdfs_files_stats.jy
index ce7ae265c..587b3b390 100755
--- a/hadoop_hdfs_files_stats.jy
+++ b/hdfs_files_stats.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2013-06-08 22:06:27 +0100 (Sat, 08 Jun 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -15,7 +15,7 @@ from __future__ import print_function
from __future__ import unicode_literals
__author__ = 'Hari Sekhon'
-__version__ = '0.9.1'
+__version__ = '0.9.2'
# pylint: disable=wrong-import-position
import os
@@ -55,7 +55,7 @@ def usage(*msg):
if msg:
printerr(msg)
die("""
-Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
================================================================================
%s - version %s
@@ -146,7 +146,7 @@ class HDFSBlockCounter(object):
try:
self.client = self.filesystem.getClient()
except:
- raise Exception, "Failed to create hdfs client: %s" % get_jython_exception()
+ raise Exception("Failed to create hdfs client: %s" % get_jython_exception())
#in = client.DFSDataInputStream(self.filehandle)
@@ -178,7 +178,7 @@ class HDFSBlockCounter(object):
printerr("Failed to get HDFS path object for file " + filename, 1)
continue
if not self.filesystem.exists(path):
- #raise IOError, "HDFS File not found: %s" % filename
+ #raise IOError("HDFS File not found: %s" % filename)
printerr("HDFS file/dir not found: %s" % filename, 1)
continue
self.recurse_path(filename, path)
@@ -213,7 +213,7 @@ class HDFSBlockCounter(object):
except:
printerr(get_jython_exception().split("\n")[0], 1)
else:
- raise IOError, ">>> %s is not a file or directory" % filename
+ raise IOError(">>> %s is not a file or directory" % filename)
def print_file_block_counts(self, filename, path):
@@ -226,9 +226,9 @@ class HDFSBlockCounter(object):
try:
self.filehandle = self.client.open(filename)
except:
- raise IOError, "Failed to get client filehandle to HDFS file %s: %s" % (filename, get_jython_exception())
+ raise IOError("Failed to get client filehandle to HDFS file %s: %s" % (filename, get_jython_exception()))
if self.filehandle is None:
- raise IOError, "Failed to get client filehandle to HDFS file %s" % filename
+ raise IOError("Failed to get client filehandle to HDFS file %s" % filename)
try:
filestatus = self.filesystem.getFileStatus(path)
diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py
new file mode 100755
index 000000000..1f348556d
--- /dev/null
+++ b/hdfs_find_replication_factor_1.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2018-11-28 16:37:00 +0000 (Wed, 28 Nov 2018)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to find HDFS file with replication factor 1
+
+These cause problems because taking a single datanode offline may result in alerts for files with missing blocks
+
+Uses any arguments are directory tree paths to starting scanning down. If no argument paths are given, searches under
+top level directory /
+
+Uses Hadoop configuration files it expects to find in $HADOOP_HOME/conf to auto-detect
+NameNodes HA, Kerberos etc (just kinit first)
+
+Optionally resets such files back to replication factor 3 if specifying --set-replication-factor-3
+
+Tested on Hadoop 2.7 on HDP 2.6 with Kerberos
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+import time
+import traceback
+import krbV
+import snakebite
+from snakebite.client import AutoConfigClient
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, log_option, validate_int
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3'
+
+
+class HdfsFindReplicationFactor1(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HdfsFindReplicationFactor1, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.path_list = None
+ self.replication_factor = None
+
+ def add_options(self):
+ super(HdfsFindReplicationFactor1, self).add_options()
+ self.add_opt('--hadoop-home',
+ help='Sets $HADOOP_HOME, expects to find config in $HADOOP_HOME/conf, ' + \
+ 'otherwise inherits from environment or tries default paths')
+ self.add_opt('--set-replication', metavar='N', type=int,
+ help='Resets any files with replication factor 1 back to this replication factor (optional)')
+
+ def process_options(self):
+ super(HdfsFindReplicationFactor1, self).process_options()
+ self.path_list = self.args
+ if not self.path_list:
+ self.path_list = ['/']
+ self.replication_factor = self.get_opt('set_replication')
+ if self.replication_factor is not None:
+ validate_int(self.replication_factor, 'set replication', 2, 5)
+ hadoop_home = self.get_opt('hadoop_home')
+ if hadoop_home is not None:
+ os.environ['HADOOP_HOME'] = hadoop_home
+ hadoop_home_env = os.getenv('HADOOP_HOME')
+ log_option('HADOOP_HOME', hadoop_home_env)
+ if hadoop_home_env:
+ log.info('will search for Hadoop config in %s/conf', hadoop_home_env)
+
+ def run(self):
+ log.info('initiating snakebite hdfs client')
+ try:
+ client = AutoConfigClient()
+ except krbV.Krb5Error as _: # pylint: disable=no-member
+ if self.verbose:
+ print('', file=sys.stderr)
+ print(_, file=sys.stderr)
+ start_time = time.time()
+ dir_count = 0
+ file_count = 0
+ repl1_count = 0
+ for path in self.path_list:
+ try:
+ result_list = client.ls([path], recurse=True, include_toplevel=True, include_children=True)
+ for result in result_list:
+ if self.verbose and (dir_count + file_count) % 100 == 0:
+ print('.', file=sys.stderr, end='')
+ if result['block_replication'] == 0:
+ dir_count += 1
+ continue
+ file_count += 1
+ if result['block_replication'] == 1:
+ file_path = result['path']
+ repl1_count += 1
+ if self.verbose:
+ print('', file=sys.stderr)
+ print(file_path)
+ if self.replication_factor:
+ log.info('setting replication factor to %s on %s', self.replication_factor, file_path)
+ # returns a generator so must evaluate in order to actually execute
+ # otherwise you find there is no effect on the replication factor
+ for _ in client.setrep([file_path], self.replication_factor, recurse=False):
+ if 'result' not in _:
+ print('WARNING: result field not found in setrep result: {}'.format(_),
+ file=sys.stderr)
+ continue
+ if not _['result']:
+ print('WARNING: failed to setrep: {}'.format(_))
+ except (snakebite.errors.FileNotFoundException, snakebite.errors.RequestError) as _:
+ if self.verbose:
+ print('', file=sys.stderr)
+ print(_, file=sys.stderr)
+ if self.verbose:
+ print('', file=sys.stderr)
+ secs = int(time.time() - start_time)
+ print('\nCompleted in {} secs\n'.format(secs), file=sys.stderr)
+ print('{} files with replication factor 1 out of {} files in {} dirs'\
+ .format(repl1_count, file_count, dir_count), file=sys.stderr)
+
+
+if __name__ == '__main__':
+ HdfsFindReplicationFactor1().main()
diff --git a/hadoop_hdfs_time_block_reads.jy b/hdfs_time_block_reads.jy
similarity index 92%
rename from hadoop_hdfs_time_block_reads.jy
rename to hdfs_time_block_reads.jy
index a212c3e6b..ab1812a23 100755
--- a/hadoop_hdfs_time_block_reads.jy
+++ b/hdfs_time_block_reads.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2013-06-08 22:06:27 +0100 (Sat, 08 Jun 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -23,12 +23,12 @@ from __future__ import print_function
from __future__ import unicode_literals
__author__ = 'Hari Sekhon'
-__version__ = '0.9.0'
+__version__ = '0.9.1'
import os
usage_msg = """
-Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
================================================================================
%s - version %s
@@ -95,7 +95,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), 'pylib'))
try:
from harisekhon.utils import jython_only, die, printerr, ERRORS, \
isJavaOOM, java_oom_fix_msg, log_jython_exception, get_jython_exception
-except ImportError, e:
+except ImportError as e:
print('module import failed: %s' % e, file=sys.stderr)
sys.exit(4)
@@ -105,14 +105,14 @@ jython_only()
import jarray
try:
from java.nio import ByteBuffer
-except ImportError, e:
+except ImportError as e:
die("Couldn't find java.nio class, not running inside Jython?")
try:
from org.apache.hadoop.conf import Configuration
#from org.apache.hadoop.fs import FileSystem
from org.apache.hadoop.fs import Path
from org.apache.hadoop.hdfs import DistributedFileSystem
-except ImportError, e:
+except ImportError as e:
die("Couldn't find Hadoop Java classes, try: jython -J-cp `hadoop classpath` hadoop_hdfs_time_block_reads.jy ")
def usage(*msg):
@@ -127,7 +127,7 @@ def main():
try:
opts, args = getopt.gnu_getopt(sys.argv[1:], "hamo", ["help", "usage", "all-blocks", "multiple-blocks", "one-block-per-DN"])
- except getopt.GetoptError, e:
+ except getopt.GetoptError as e:
usage("error: %s" % e)
all_blocks = False
multiple_blocks = False
@@ -153,10 +153,10 @@ def main():
filelist = sorted(filelist)
try:
HDFSBlockReader(multiple_blocks, one_block_per_DN).fetchFilelistBlocksTimed(filelist)
- except KeyboardInterrupt, e:
+ except KeyboardInterrupt as e:
printerr("Caught Control-C...", 1)
sys.exit(ERRORS["OK"])
- except Exception, e:
+ except Exception as e:
printerr("Error running HDFSBlockReader: %s" % e, 1)
if isJavaOOM(e.message):
printerr(java_oom_fix_msg())
@@ -206,7 +206,7 @@ class HDFSBlockReader:
try:
self.client = self.fs.getClient()
except:
- raise Exception, "Failed to create hdfs client: %s" % get_jython_exception()
+ raise Exception("Failed to create hdfs client: %s" % get_jython_exception())
#in = client.DFSDataInputStream(self.fh)
@@ -214,7 +214,7 @@ class HDFSBlockReader:
""" Return the path object for a given filename """
try:
path = Path(filename)
- except Exception, e:
+ except Exception as e:
return None
if path:
return path
@@ -238,7 +238,7 @@ class HDFSBlockReader:
printerr("Failed to get HDFS path object for file " + filename, 1)
continue
if not self.fs.exists(path):
- #raise IOError, "HDFS File not found: %s" % filename
+ #raise IOError("HDFS File not found: %s" % filename)
printerr("HDFS file/dir not found: %s" % filename, 1)
continue
self.recurse_path(filename, path)
@@ -292,7 +292,7 @@ class HDFSBlockReader:
if not self.multiple_files:
print("Reading File %s\n" % filename)
self.fetchFileBlocksTimed(filename, path)
- except Exception, e:
+ except Exception as e:
printerr(e, 1)
elif self.fs.isDirectory(path):
# even if there is only one file under the whole directory tree since it's now different to the specified arg we should print it
@@ -308,7 +308,7 @@ class HDFSBlockReader:
except:
printerr(get_jython_exception().split("\n")[0], 1)
else:
- raise IOError, ">>> %s is not a file or directory" % filename
+ raise IOError(">>> %s is not a file or directory" % filename)
def fetchFileBlocksTimed(self, filename, path):
@@ -323,16 +323,16 @@ class HDFSBlockReader:
try:
self.fh = self.client.open(filename)
except:
- raise IOError, "Failed to get client filehandle to HDFS file %s: %s" % (filename, get_jython_exception())
+ raise IOError("Failed to get client filehandle to HDFS file %s: %s" % (filename, get_jython_exception()))
if self.fh == None:
- raise IOError, "Failed to get client filehandle to HDFS file %s" % filename
+ raise IOError("Failed to get client filehandle to HDFS file %s" % filename)
while True:
self.block_num += 1
try:
block_locations = self.fs.getFileBlockLocations(path, self.offset, self.length)
except:
- raise IOError, "Failed to get block locations for %sblock %d (offset %d, length %d): %s" % (self.fileoutput, self.block_num, self.offset, self.length, get_jython_exception())
+ raise IOError("Failed to get block locations for %sblock %d (offset %d, length %d): %s" % (self.fileoutput, self.block_num, self.offset, self.length, get_jython_exception()))
if not block_locations:
return
for block in block_locations:
@@ -344,11 +344,11 @@ class HDFSBlockReader:
return
self.offset = self.block_offset + self.block_length
except:
- raise IOError, "Failed to get %sblock %d offset/length: %s" % (self.fileoutput, self.block_num, get_jython_exception())
+ raise IOError("Failed to get %sblock %d offset/length: %s" % (self.fileoutput, self.block_num, get_jython_exception()))
try:
self.readBlockFromDNs()
- except Exception, e:
- raise IOError, "Failed to read %sblock %d (offset %d, length %d) from datanodes: %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length, e)
+ except Exception as e:
+ raise IOError("Failed to read %sblock %d (offset %d, length %d) from datanodes: %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length, e))
def readBlockFromDNs(self):
""" Read current block self.offset to self.length from all datanodes """
@@ -375,10 +375,10 @@ class HDFSBlockReader:
# self.dn must be populated by here or something is wrong
self.dn = self.fh.getCurrentDatanode()
if self.dn == None:
- raise IOError, "Failed to get current DataNode for %sblock %s offset %s length %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length)
+ raise IOError("Failed to get current DataNode for %sblock %s offset %s length %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length))
self.host = self.dn.getHostName()
if not self.host:
- raise Exception, "Failed to get current DataNode host name for %sblock %s offset %s length %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length)
+ raise Exception("Failed to get current DataNode host name for %sblock %s offset %s length %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length))
# I've come back to the first DN, done with this block
if self.host in self.block_read_DNs:
return
@@ -419,8 +419,8 @@ class HDFSBlockReader:
# else:
# printerr("ERROR: Failed to read bytes from %sblock %d (offset %d, length %d) [datanode info not available]" % (self.fileoutput, self.block_num, self.block_offset, self.block_length))
# return
- except Exception, e:
- raise Exception, e
+ except Exception as e:
+ raise Exception(e)
except:
self.dn = self.fh.getCurrentDatanode()
if self.dn:
@@ -429,7 +429,7 @@ class HDFSBlockReader:
self.nodes_failed_reads.add(self.host)
print("Error: failed to read %sblock %d (offset %d, length %d) from rack %s datanode %s: %s" % (self.fileoutput, self.block_num, self.block_offset, self.block_length, self.rack, self.host, get_jython_exception()))
else:
- raise IOError, get_jython_exception()
+ raise IOError(get_jython_exception())
return
end_time = time.time()
time_taken = end_time - start_time
diff --git a/headtail.py b/headtail.py
index ff061445a..ec21a3f5c 100755
--- a/headtail.py
+++ b/headtail.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-07 22:57:18 +0000 (Thu, 07 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish # pylint: disable=line-too-long
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -41,7 +41,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.3.1'
+__version__ = '0.3.2'
class HeadTail(CLI):
@@ -60,6 +60,7 @@ def __init__(self):
self.sep = '-' * 80
self.docsep = '=' * 80
self.quiet = False
+ self.timeout_default = None
def add_options(self):
#self.timeout_default = 300
diff --git a/hexanonymize.py b/hexanonymize.py
new file mode 100755
index 000000000..cac35e6a0
--- /dev/null
+++ b/hexanonymize.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-01-02 17:08:32 +0000 (Thu, 02 Jan 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to anonymize hex input but keeping the structure of the positions of numbers and digits
+
+Useful to retain the structure of ID formats
+
+Reads any given files or standard input and replaces each hex character with an incrementing number of letter (a-f)
+printing to stdout for piping or redirecting to a file as per unix filter command standards
+
+Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from
+standard input
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import os
+import sys
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+
+class HexAnonymize(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HexAnonymize, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.preserve_case = False
+ self.only_hex_alphas = False
+
+ def add_options(self):
+ super(HexAnonymize, self).add_options()
+ self.add_opt('-c', '--case', action='store_true', help='Preserve case')
+ self.add_opt('-o', '--hex-only', action='store_true',
+ help='Only replace hex alpha chars (A-F, a-f), otherwise replaces all alphanumerics for safety')
+
+ def process_options(self):
+ super(HexAnonymize, self).process_options()
+ self.preserve_case = self.get_opt('case')
+ self.only_hex_alphas = self.get_opt('hex_only')
+
+ def hexanonymize(self, filehandle):
+ preserve_case = self.preserve_case
+ only_hex_alphas = self.only_hex_alphas
+ hex_alphas = ['a', 'b', 'c', 'd', 'e', 'f']
+ for line in filehandle:
+ integer = 1
+ letter = 'a'
+ for char in line:
+ if char.isdigit():
+ char = integer
+ integer += 1
+ if integer > 9:
+ integer = 0
+ elif (not only_hex_alphas and char.isalpha()) or char.lower() in hex_alphas:
+ if preserve_case and char.isupper():
+ char = letter.upper()
+ else:
+ char = letter
+ letter = chr(ord(char) + 1)
+ if letter.lower() not in hex_alphas:
+ letter = 'a'
+ print(char, end='')
+
+
+ def run(self):
+ if not self.args:
+ self.args.append('-')
+ for arg in self.args:
+ if arg == '-':
+ continue
+ if not os.path.exists(arg):
+ print("'%s' not found" % arg)
+ sys.exit(1)
+ for arg in self.args:
+ if arg == '-':
+ self.hexanonymize(sys.stdin)
+ else:
+ with open(arg) as filehandle:
+ self.hexanonymize(filehandle)
+
+
+if __name__ == '__main__':
+ HexAnonymize().main()
diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py
new file mode 100755
index 000000000..d416f9379
--- /dev/null
+++ b/hive_compute_table_stats.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to a HiveServer2 and compute optimization statistics on all tables,
+or only those matching given db / table / partition value regexes
+
+Tested on CDH 5.10, Hive 1.1.0 with Kerberos
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import logging
+import os
+import re
+import socket
+import sys
+# pylint: disable=import-error
+from impala.dbapi import connect
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+logging.basicConfig()
+log = logging.getLogger(os.path.basename(sys.argv[0]))
+
+host_envs = [
+ 'HIVESERVER2_HOST',
+ 'HIVE_HOST',
+ 'HOST'
+]
+
+port_envs = [
+ 'HIVESERVER2_PORT',
+ 'HIVE_PORT',
+ 'PORT'
+]
+
+def getenvs(keys, default=None):
+ for key in keys:
+ value = os.getenv(key)
+ if value:
+ return value
+ return default
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Computes statistics on all Hive tables/partitions matching database / table / partition regexes")
+ parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\
+ help='HiveServer2 host ' + \
+ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')')
+ parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000),
+ help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')')
+ parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)')
+ parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)')
+ parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)')
+ parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)')
+ parser.add_argument('-n', '--krb5-service-name', default='hive',
+ help='Service principal (default: \'hive\')')
+ parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL')
+ parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode')
+ args = parser.parse_args()
+
+ if args.verbose:
+ log.setLevel(logging.INFO)
+
+ return args
+
+def connect_db(args, database):
+ auth_mechanism = None
+ if args.kerberos:
+ auth_mechanism = 'GSSAPI'
+
+ log.info('connecting to %s:%s database %s', args.host, args.port, database)
+ return connect(
+ host=args.host,
+ port=args.port,
+ auth_mechanism=auth_mechanism,
+ use_ssl=args.ssl,
+ #user=user,
+ #password=password,
+ database=database,
+ kerberos_service_name=args.krb5_service_name
+ )
+
+def main():
+ args = parse_args()
+
+ try:
+ database_regex = re.compile(args.database, re.I)
+ table_regex = re.compile(args.table, re.I)
+ partition_regex = re.compile(args.partition, re.I)
+ except re.error as _:
+ log.error('error in provided regex: %s', _)
+ sys.exit(3)
+
+ conn = connect_db(args, None)
+
+ log.info('querying databases')
+ with conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ if not database_regex.search(database):
+ log.debug("skipping database '%s', does not match regex '%s'", database, args.database)
+ continue
+ log.info('querying tables for database %s', database)
+ #db_conn = connect_db(args, database)
+ #with db_conn.cursor() as table_cursor:
+ with conn.cursor() as table_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use {}'.format(database))
+ table_cursor.execute('show tables')
+ for table_row in table_cursor:
+ table = table_row[0]
+ if not table_regex.search(table):
+ log.debug("skipping database '%s' table '%s', does not match regex '%s'", \
+ database, table, args.table)
+ continue
+ compute_table_stats(conn, args, database, table, partition_regex)
+
+def compute_table_stats(conn, args, database, table, partition_regex):
+ log.info("getting partitions for database '%s' table '%s'", database, table)
+ partitions_found = False
+ with conn.cursor() as partition_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('use {}'.format(database))
+ partition_cursor.execute('show partitions {}'.format(database))
+ for partitions_row in partition_cursor:
+ partition_key = partitions_row[0]
+ partition_value = partitions_row[1]
+ partitions_found = True
+ if not partition_regex.match(partition_value):
+ # pylint: disable=logging-not-lazy
+ log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " +
+ "value does not match regex '%s'",
+ database,
+ table,
+ partition_key,
+ partition_value,
+ args.partition)
+ continue
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('ANALYZE TABLE {db}.{table} PARTITION({key}={value}) COMPUTE STATISTICS'\
+ .format(db=database, table=table, key=partition_key, value=partition_value))
+ if not partitions_found:
+ log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table)
+ with conn.cursor() as table_cursor:
+ log.info("running compute stats on table '%s'", table)
+ # doesn't support parameterized query quoting from dbapi spec
+ table_cursor.execute('ANALYZE TABLE {db}.{table} COMPUTE STATISTICS'.format(db=database, table=table))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/hive_foreach_table.py b/hive_foreach_table.py
new file mode 100755
index 000000000..9d5abf6d6
--- /dev/null
+++ b/hive_foreach_table.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and execute a query for each table in each database,
+or only those matching given db / table regexes
+
+Useful for getting row counts of all tables or analyzing tables:
+
+eg.
+
+hive_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}'
+hive_foreach_table.py --query 'ANALYZE TABLE {db}.{table} COMPUTE STATS'
+
+or just for today's partition:
+
+hive_foreach_table.py --query "ANALYZE TABLE {db}.{table} PARTITION(date=$(date '+%Y-%m-%d')) COMPUTE STATS"
+
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+import impala
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_regex
+ from hive_impala_cli import HiveImpalaCLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.2'
+
+
+class HiveForEachTable(HiveImpalaCLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveForEachTable, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # self.query can be pre-defined in which case subclassed programs won't expose --query option
+ self.query = None
+ self.database = None
+ self.table = None
+ self.partition = None
+ self.ignore_errors = False
+ self.table_count = 0
+
+ def add_options(self):
+ super(HiveForEachTable, self).add_options()
+ # allow subclassing to pre-define query and not expose option in that case
+ if self.query is None:
+ self.add_opt('-q', '--query', help='Query or statement to execute for each table' + \
+ ' (replaces {db} and {table} in the query string with each table and its database)')
+ self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)')
+ self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)')
+ #self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)')
+#
+# ignore tables that fail with errors like:
+#
+# Hive (CDH has MR, no tez):
+#
+# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long
+#
+# Impala:
+#
+# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table ''
+# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table ''
+#
+ self.add_opt('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue')
+
+ def process_options(self):
+ super(HiveForEachTable, self).process_options()
+ if self.query is None:
+ self.query = self.get_opt('query')
+ if not self.query:
+ self.usage('query not defined')
+ self.database = self.get_opt('database')
+ self.table = self.get_opt('table')
+ #self.partition = self.get_opt('partition')
+ self.ignore_errors = self.get_opt('ignore_errors')
+ validate_regex(self.database, 'database')
+ validate_regex(self.table, 'table')
+ #validate_regex(self.partition, 'partition')
+
+ def run(self):
+ database_regex = re.compile(self.database, re.I)
+ table_regex = re.compile(self.table, re.I)
+ #partition_regex = re.compile(self.partition, re.I)
+ conn = self.connect('default')
+ log.info('querying databases')
+ # collecting in local list because long time iteration results in
+ # impala.error.HiveServer2Error: Invalid query handle
+ databases = []
+ database_count = 0
+ with conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ database_count += 1
+ if not database_regex.search(database):
+ log.debug("skipping database '%s', does not match regex '%s'", database, self.database)
+ continue
+ databases.append(database)
+ log.info('%s/%s databases selected', len(databases), database_count)
+ for database in databases:
+ self.process_database(database, table_regex)
+ log.info('processed %s databases, %s tables', database_count, self.table_count)
+
+ def process_database(self, database, table_regex):
+ tables = []
+ table_count = 0
+ log.info("querying tables for database '%s'", database)
+ conn = self.connect(database)
+ with conn.cursor() as table_cursor:
+ try:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use `{}`'.format(database))
+ table_cursor.execute('show tables')
+ except impala.error.HiveServer2Error as _:
+ log.error("error querying tables for database '%s': %s", database, _)
+ if 'AuthorizationException' in str(_):
+ return
+ raise
+ for table_row in table_cursor:
+ table = table_row[0]
+ table_count += 1
+ if not table_regex.search(table):
+ log.debug("skipping database '%s' table '%s', does not match regex '%s'", \
+ database, table, self.table)
+ continue
+ tables.append(table)
+ log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database)
+ for table in tables:
+ try:
+ query = self.query.format(db='`{}`'.format(database),
+ table='`{}`'.format(table))
+ except KeyError as _:
+ if _ == 'db':
+ query = self.query.format(table='`{}`'.format(table))
+ else:
+ raise
+ try:
+ self.execute(conn, database, table, query)
+ self.table_count += 1
+ except Exception as _:
+ if self.ignore_errors:
+ log.error("database '%s' table '%s': %s", database, table, _)
+ continue
+ raise
+
+ @staticmethod
+ def execute(conn, database, table, query):
+ try:
+ log.info(' %s.%s - running %s', database, table, query)
+ with conn.cursor() as query_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ query_cursor.execute(query)
+ for result in query_cursor:
+ print('{db}.{table}\t{result}'.format(db=database, table=table, \
+ result='\t'.join([str(_) for _ in result])))
+ #except (impala.error.OperationalError, impala.error.HiveServer2Error) as _:
+ # log.error(_)
+ except impala.error.ProgrammingError as _:
+ log.error(_)
+ # COMPUTE STATS returns no results
+ if 'Trying to fetch results on an operation with no results' not in str(_):
+ raise
+
+
+if __name__ == '__main__':
+ HiveForEachTable().main()
diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py
new file mode 100755
index 000000000..ea41f3753
--- /dev/null
+++ b/hive_schemas_csv.py
@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and dump all the schemas, tables and columns out in CSV format to stdout
+
+In practice Hive is much more reliable than Impala for dumping masses of schema
+
+Impala appears faster initially but then slows down more than Hive and hits things query handle errors
+under sustained load of extracting large amounts of schema information
+
+There is also a risk that Impala's metadata may be out of date, so Hive is strongly preferred for this
+
+
+CSV format:
+
+database,table,column,type
+
+
+I recommend generating quoted csv because you may encounter Hive data types such as decimal(15,2)
+which would cause incorrect field splitting, you can disable by setting --quotechar='' to blank but
+if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will
+raise a traceback to tell you to set one (eg. --escapechar='\\')
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import csv
+import os
+import sys
+import impala
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_impala_cli import HiveImpalaCLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.1'
+
+
+class HiveSchemasCSV(HiveImpalaCLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveSchemasCSV, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.delimiter = None
+ self.quotechar = None
+ self.escapechar = None
+ self.table_count = 0
+ self.column_count = 0
+ self.ignore_errors = False
+ self.csv_writer = None
+ self.conn = None
+
+ def add_options(self):
+ super(HiveSchemasCSV, self).add_options()
+ # must set type to str otherwise csv module gives this error on Python 2.7:
+ # TypeError: "delimiter" must be string, not unicode
+ # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals'
+ # breaks this - might break in Python 3 if the impyla module doesn't fix behaviour
+ self.add_opt('-d', '--delimiter', default=',', type=str, help='Delimiter to use (default: ,)')
+ self.add_opt('-Q', '--quotechar', default='"', type=str,
+ help='Generate quoted CSV (recommended, default is double quote \'"\')')
+ self.add_opt('-E', '--escapechar', help='Escape char if needed')
+#
+# ignore tables that fail with errors like:
+#
+# Hive (CDH has MR, no tez):
+#
+# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long
+#
+# Impala:
+#
+# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table ''
+# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table ''
+#
+ self.add_opt('-e', '--ignore-errors', action='store_true',
+ help='Ignore individual table schema listing errors (Impala often has table metadata errors)')
+
+ def process_options(self):
+ super(HiveSchemasCSV, self).process_options()
+ self.delimiter = self.get_opt('delimiter')
+ self.quotechar = self.get_opt('quotechar')
+ self.escapechar = self.get_opt('escapechar')
+ self.ignore_errors = self.get_opt('ignore_errors')
+
+ def run(self):
+
+ self.conn = self.connect('default')
+
+ quoting = csv.QUOTE_ALL
+ if self.quotechar == '':
+ quoting = csv.QUOTE_NONE
+ fieldnames = ['database', 'table', 'column', 'type']
+ self.csv_writer = csv.DictWriter(sys.stdout,
+ delimiter=self.delimiter,
+ quotechar=self.quotechar,
+ escapechar=self.escapechar,
+ quoting=quoting,
+ fieldnames=fieldnames)
+ self.csv_writer.writeheader()
+ log.info('querying databases')
+ databases = []
+ database_count = 0
+ with self.conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ database_count += 1
+ databases.append(database)
+ log.info('found %s databases', database_count)
+ for database in databases:
+ self.process_database(database)
+ log.info('databases: %s, tables: %s, columns: %s', database_count, self.table_count, self.column_count)
+
+ def process_database(self, database):
+ log.info("querying tables for database '%s'", database)
+ tables = []
+ with self.conn.cursor() as table_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use `{}`'.format(database))
+ table_cursor.execute('show tables')
+ for table_row in table_cursor:
+ table = table_row[0]
+ self.table_count += 1
+ tables.append(table)
+ log.info("found %s tables in database '%s'", len(tables), database)
+ for table in tables:
+ try:
+ self.process_table(database, table)
+ except impala.error.HiveServer2Error as _:
+ if self.ignore_errors:
+ log.error("database '%s' table '%s': %s", database, table, _)
+ continue
+ raise
+
+ def process_table(self, database, table):
+ log.info("describing table '%s.%s'", database, table)
+ with self.conn.cursor() as column_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #column_cursor.execute('use %(database)s', {'database': database})
+ #column_cursor.execute('describe %(table)s', {'table': table})
+ column_cursor.execute('use `{}`'.format(database))
+ column_cursor.execute('describe `{}`'.format(table))
+ column_count = 0
+ for column_row in column_cursor:
+ column = column_row[0]
+ column_type = column_row[1]
+ column_count += 1
+ self.csv_writer.writerow({'database': database,
+ 'table': table,
+ 'column': column,
+ 'type': column_type})
+ log.info("found %s columns in table '%s.%s'", column_count, database, table)
+ self.column_count += column_count
+
+
+if __name__ == '__main__':
+ HiveSchemasCSV().main()
diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py
new file mode 100755
index 000000000..d6cb4ce68
--- /dev/null
+++ b/hive_tables_column_counts.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and count the number of columns for each table in each database,
+or only those matching given db / table regexes
+
+You can also get this from the schemas.csv output generated by hive_schemas_csv.py, eg.
+
+ tail -n +2 hive_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveTablesColumnCounts(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesColumnCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # not needed, here to suppress --query CLI option
+ self.database = None
+ self.table = None
+ self.ignore_errors = False
+
+ # discarding last param query
+ def execute(self, conn, database, table, query):
+ column_count = 0
+ log.info("describing table '%s.%s'", database, table)
+ with conn.cursor() as column_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #column_cursor.execute('use %(database)s', {'database': database})
+ #column_cursor.execute('describe %(table)s', {'table': table})
+ column_cursor.execute('use `{}`'.format(database))
+ # don't use desc here, Impala doesn't support it and would break subclass
+ column_cursor.execute('describe `{}`'.format(table))
+ for _ in column_cursor:
+ #column = _[0]
+ #column_type = _[1]
+ column_count += 1
+ print('{db}.{table}\t{column_count}'.format(db=database, table=table, column_count=column_count))
+
+
+if __name__ == '__main__':
+ HiveTablesColumnCounts().main()
diff --git a/hive_tables_list.py b/hive_tables_list.py
new file mode 100755
index 000000000..64675c61f
--- /dev/null
+++ b/hive_tables_list.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and list all databases and tables
+
+TSV Output format:
+
+
+
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+
+class HiveTablesList(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesList, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # here merely to suppress --query CLI option
+
+ def execute(self, conn, database, table, query):
+ print('{}\t{}'.format(database, table))
+
+
+if __name__ == '__main__':
+ HiveTablesList().main()
diff --git a/hive_tables_locations.py b/hive_tables_locations.py
new file mode 100755
index 000000000..403e07443
--- /dev/null
+++ b/hive_tables_locations.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and list the locations of all tables in all databases,
+or only those matching given db / table regexes
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_metadata import HiveTablesMetadata
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveTablesLocations(HiveTablesMetadata):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesLocations, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.field = 'Location'
+
+
+if __name__ == '__main__':
+ HiveTablesLocations().main()
diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py
new file mode 100755
index 000000000..d15d62feb
--- /dev/null
+++ b/hive_tables_metadata.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and print the first matching DDL metadata field (eg. 'Location')
+for each table in each database, or only those matching given db / table regexes
+
+Examples (fields are case sensitive regex and return N/A without match):
+
+./hive_tables_metadata.py --field Location ...
+./hive_tables_metadata.py --field SerDe ...
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_regex
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.1'
+
+
+class HiveTablesMetadata(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesMetadata, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'describe formatted {table}'
+ self.field = None
+
+ def add_options(self):
+ # Python 2.x
+ super(HiveTablesMetadata, self).add_options()
+ # Python 3.x
+ # super().__init__()
+ if self.field is None:
+ self.add_opt('-f', '--field', help='Table DDL metadata field to return for each table (required)')
+
+ def process_options(self):
+ # Python 2.x
+ super(HiveTablesMetadata, self).process_options()
+ # Python 3.x
+ # super().__init__()
+ if self.field is None:
+ self.field = self.get_opt('field')
+ if not self.field:
+ self.usage('--field not specified')
+ validate_regex(self.field, 'field')
+ self.field = re.compile(self.field)
+
+ # discard last param query and construct our own based on the table DDL of cols
+ def execute(self, conn, database, table, query):
+ log.info("describing table '%s.%s'", database, table)
+ field = 'N/A'
+ with conn.cursor() as table_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ #table_cursor.execute('describe %(table)s', {'table': table})
+ table_cursor.execute('use `{}`'.format(database))
+ table_cursor.execute(query.format(table=table))
+ for row in table_cursor:
+ if self.field.search(row[0]):
+ field = row[1]
+ break
+ print('{db}.{table}\t{field}'.format(db=database, table=table, field=field))
+
+
+if __name__ == '__main__':
+ HiveTablesMetadata().main()
diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py
new file mode 100755
index 000000000..207fcba3f
--- /dev/null
+++ b/hive_tables_null_columns.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and find tables with columns containing only NULLs
+for all tables in all databases, or only those matching given db / table regexes
+
+Describes each table, constructs a complex query to check each column individually for containing only NULLs,
+and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveTablesNullColumns(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesNullColumns, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option
+ self.database = None
+ self.table = None
+ #self.partition = None
+ self.ignore_errors = False
+
+ # discard last param query and construct our own based on the table DDL of cols
+ def execute(self, conn, database, table, _query):
+ sum_part = ''
+ columns = []
+ log.info("describing table '%s.%s'", database, table)
+ with conn.cursor() as column_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #column_cursor.execute('use %(database)s', {'database': database})
+ #column_cursor.execute('describe %(table)s', {'table': table})
+ column_cursor.execute('use `{}`'.format(database))
+ column_cursor.execute('describe `{}`'.format(table))
+ for column_row in column_cursor:
+ column = column_row[0]
+ #column_type = column_row[1]
+ columns.append(column)
+ sum_part = ', '.join(
+ ['IF(SUM(IF(`{col}` IS NULL, 1, 0)) = COUNT(*), 1, 0) as `{col}`'.format(col=column) \
+ for column in columns]
+ )
+ query = "SELECT {sum_part} FROM `{db}`.`{table}` WHERE `"\
+ .format(sum_part=sum_part, db=database, table=table) + \
+ "` IS NULL OR `".join(columns) + "` IS NULL"
+ self.check_table_for_nulls(conn, database, table, columns, query)
+
+ @staticmethod
+ def check_table_for_nulls(conn, database, table, columns, query):
+ with conn.cursor() as table_cursor:
+ log.debug('executing query: %s', query)
+ table_cursor.execute(query)
+ cols_with_nulls = []
+ for result in table_cursor:
+ # tuple of ints (0, 0, 0, .... N) - one per column
+ for index in range(len(list(result))):
+ col_result = result[index]
+ if col_result > 0:
+ cols_with_nulls.append(columns[index])
+ num_cols = len(cols_with_nulls)
+ total_cols = len(columns)
+ if cols_with_nulls:
+ print('WARNING: {db}.{table} has {num}/{total} columns with only NULLs: {cols}'\
+ .format(db=database,
+ table=table,
+ num=num_cols,
+ total=total_cols,
+ cols=', '.join(sorted(cols_with_nulls))
+ )
+ )
+ else:
+ print('OK: {db}.{table} has {num}/{total} columns with only nulls'\
+ .format(db=database, table=table, num=num_cols, total=total_cols))
+
+
+if __name__ == '__main__':
+ HiveTablesNullColumns().main()
diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py
new file mode 100755
index 000000000..f1a5752f1
--- /dev/null
+++ b/hive_tables_null_rows.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and count number of rows with only NULLs in all columns
+for each table in each database, or only those matching given db / table regexes
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.6.0'
+
+
+class HiveTablesNullRows(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesNullRows, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option
+ self.database = None
+ self.table = None
+ #self.partition = None
+ self.ignore_errors = False
+
+ # discard last param query and construct our own based on the table DDL of cols
+ def execute(self, conn, database, table, _):
+ columns = []
+ log.info("describing table '%s.%s'", database, table)
+ with conn.cursor() as column_cursor:
+ # impala library doesn't support parameterized query quoting from dbapi spec
+ #column_cursor.execute('use %(database)s', {'database': database})
+ #column_cursor.execute('describe %(table)s', {'table': table})
+ column_cursor.execute('use `{}`'.format(database))
+ column_cursor.execute('describe `{}`'.format(table))
+ for column_row in column_cursor:
+ column = column_row[0]
+ #column_type = column_row[1]
+ columns.append(column)
+ query = self.generate_sql(database, table, columns)
+ with conn.cursor() as table_cursor:
+ log.debug('executing query: %s', query)
+ table_cursor.execute(query)
+ for result in table_cursor:
+ count = result[0]
+ print('{db}.{table}\t{count}'.format(db=database, table=table, count=count))
+
+ @staticmethod
+ def generate_sql(database, table, columns):
+ sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\
+ .format(db=database, table=table) + \
+ "` IS NULL AND `".join(columns) + "` IS NULL"
+ return sql
+
+
+if __name__ == '__main__':
+ HiveTablesNullRows().main()
diff --git a/hive_tables_row_column_counts.py b/hive_tables_row_column_counts.py
new file mode 100755
index 000000000..c2a5ec7e9
--- /dev/null
+++ b/hive_tables_row_column_counts.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and count the number of rows and columns for each table
+in each database, or only those matching given db / table regexes
+
+Output format:
+
+ .
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveTablesRowColumnCounts(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesRowColumnCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.query = 'placeholder' # not needed, here to suppress --query CLI option
+ self.database = None
+ self.table = None
+ self.ignore_errors = False
+
+ # discarding last param query
+ def execute(self, conn, database, table, query):
+ row_count = None
+ column_count = 0
+ log.info("describing table '%s.%s'", database, table)
+ with conn.cursor() as cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #cursor.execute('use %(database)s', {'database': database})
+ #cursor.execute('describe %(table)s', {'table': table})
+ cursor.execute('use `{}`'.format(database))
+ # don't use desc here, Impala doesn't support it and would break subclass
+ cursor.execute('describe `{}`'.format(table))
+ for _ in cursor:
+ #column = _[0]
+ #column_type = _[1]
+ column_count += 1
+ log.info("running SELECT COUNT(*) FROM `%s`.`%s`", database, table)
+ # doesn't support parameterized query quoting from dbapi spec
+ cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}`'.format(db=database, table=table))
+ for result in cursor:
+ assert row_count is None
+ row_count = result[0]
+ print('{db}.{table}\t{row_count}\t{column_count}'\
+ .format(db=database,
+ table=table,
+ row_count=row_count,
+ column_count=column_count))
+
+
+if __name__ == '__main__':
+ HiveTablesRowColumnCounts().main()
diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py
new file mode 100755
index 000000000..27aaa1a9c
--- /dev/null
+++ b/hive_tables_row_counts.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and get rows counts for all tables in all databases,
+or only those matching given db / table / partition value regexes
+
+Useful for reconciliations between clusters after migrations
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+import impala
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_regex
+ from hive_impala_cli import HiveImpalaCLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.6.1'
+
+
+class HiveTablesRowCounts(HiveImpalaCLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveTablesRowCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.database = None
+ self.table = None
+ self.partition = None
+ self.ignore_errors = False
+ self.table_count = 0
+
+ def add_options(self):
+ super(HiveTablesRowCounts, self).add_options()
+ self.add_opt('-d', '--database', default='.*', help='Database regex (default: .*)')
+ self.add_opt('-t', '--table', default='.*', help='Table regex (default: .*)')
+ self.add_opt('-p', '--partition', default='.*', help='Partition regex (default: .*)')
+#
+# ignore tables that fail with errors like:
+#
+# Hive (CDH has MR, no tez):
+#
+# impala.error.OperationalError: Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask # pylint: disable=line-too-long
+#
+# Impala:
+#
+# impala.error.HiveServer2Error: AnalysisException: Unsupported type 'void' in column '' of table ''
+# CAUSED BY: TableLoadingException: Unsupported type 'void' in column '' of table ''
+#
+ self.add_opt('-e', '--ignore-errors', action='store_true', help='Ignore individual table errors and continue')
+
+ def process_options(self):
+ super(HiveTablesRowCounts, self).process_options()
+ self.database = self.get_opt('database')
+ self.table = self.get_opt('table')
+ self.partition = self.get_opt('partition')
+ self.ignore_errors = self.get_opt('ignore_errors')
+ validate_regex(self.database, 'database')
+ validate_regex(self.table, 'table')
+ validate_regex(self.partition, 'partition')
+
+ def run(self):
+ database_regex = re.compile(self.database, re.I)
+ table_regex = re.compile(self.table, re.I)
+ partition_regex = re.compile(self.partition, re.I)
+ conn = self.connect('default')
+ log.info('querying databases')
+ # collecting in local list because long time iteration results in
+ # impala.error.HiveServer2Error: Invalid query handle
+ databases = []
+ database_count = 0
+ with conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ database_count += 1
+ if not database_regex.search(database):
+ log.debug("skipping database '%s', does not match regex '%s'", database, self.database)
+ continue
+ databases.append(database)
+ log.info('%s/%s databases selected', len(databases), database_count)
+ for database in databases:
+ self.process_database(conn, database, table_regex, partition_regex)
+ log.info('processed %s databases, %s tables', database_count, self.table_count)
+
+ def process_database(self, conn, database, table_regex, partition_regex):
+ tables = []
+ table_count = 0
+ log.info("querying tables for database '%s'", database)
+ with conn.cursor() as table_cursor:
+ try:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use `{}`'.format(database))
+ table_cursor.execute('show tables')
+ except impala.error.HiveServer2Error as _:
+ log.error(_)
+ if 'AuthorizationException' in str(_):
+ return
+ raise
+ for table_row in table_cursor:
+ table = table_row[0]
+ table_count += 1
+ if not table_regex.search(table):
+ log.debug("skipping database '%s' table '%s', does not match regex '%s'", \
+ database, table, self.table)
+ continue
+ tables.append(table)
+ log.info("%s/%s tables selected for database '%s'", len(tables), table_count, database)
+ for table in tables:
+ try:
+ self.get_row_counts(conn, database, table, partition_regex)
+ self.table_count += 1
+ except Exception as _:
+ # invalid query handle and similar errors happen at higher level
+ # as they are not query specific, will not be caught here so still error out
+ if self.ignore_errors:
+ log.error("database '%s' table '%s': %s", database, table, _)
+ continue
+ raise
+
+ def get_row_counts(self, conn, database, table, partition_regex):
+ log.info("getting partitions for database '%s' table '%s'", database, table)
+ with conn.cursor() as partition_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('use `{db}`'.format(db=database))
+ try:
+ partition_cursor.execute('show partitions `{table}`'.format(table=table))
+ for partitions_row in partition_cursor:
+ partition_key = partitions_row[0]
+ partition_value = partitions_row[1]
+ if not partition_regex.match(partition_value):
+ log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " +
+ "value does not match regex '%s'",
+ database,
+ table,
+ partition_key,
+ partition_value,
+ self.partition)
+ continue
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}` WHERE `{key}`={value}'\
+ .format(db=database, table=table, key=partition_key, value=partition_value))
+ for result in partition_cursor:
+ row_count = result[0]
+ print('{db}.{table}.{key}={value}\t{row_count}'.format(\
+ db=database,
+ table=table,
+ key=partition_key,
+ value=partition_value,
+ row_count=row_count))
+ except (impala.error.OperationalError, impala.error.HiveServer2Error) as _:
+ # Hive impala.error.HiveServer2Error: is not a partitioned table
+ # Impala impala.error.HiveServer2Error: Table is not partitioned
+ if 'is not a partitioned table' not in str(_) and \
+ 'Table is not partitioned' not in str(_):
+ raise
+ log.info("no partitions found for database '%s' table '%s', getting row counts for whole table",
+ database, table)
+ with conn.cursor() as table_cursor:
+ log.info("running SELECT COUNT(*) FROM `%s`.`%s`", database, table)
+ # doesn't support parameterized query quoting from dbapi spec
+ table_cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}`'.format(db=database, table=table))
+ for result in table_cursor:
+ row_count = result[0]
+ print('{db}.{table}\t{row_count}'.format(db=database, table=table, row_count=row_count))
+
+
+if __name__ == '__main__':
+ HiveTablesRowCounts().main()
diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py
new file mode 100755
index 000000000..8ea7e6b75
--- /dev/null
+++ b/hive_tables_row_counts_any_nulls.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to HiveServer2 and count number of rows with NULL in any column
+for each table in each database, or only those matching given db / table regexes
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo
+
+Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_null_rows import HiveTablesNullRows
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.6.0'
+
+
+class HiveTablesRowsWithNulls(HiveTablesNullRows):
+
+ @staticmethod
+ def generate_sql(database, table, columns):
+ # impala library doesn't support parameterized query quoting from dbapi spec
+ sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\
+ .format(db=database, table=table) + \
+ "` IS NULL OR `".join(columns) + "` IS NULL"
+ return sql
+
+
+if __name__ == '__main__':
+ HiveTablesRowsWithNulls().main()
diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py
new file mode 100755
index 000000000..04d462e16
--- /dev/null
+++ b/impala_compute_table_stats.py
@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to a Impalad and compute optimization statistics on all tables,
+or only those matching given db / table / partition value regexes
+
+Tested on CDH 5.10, Impala 1.1.0 with Kerberos
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import logging
+import os
+import re
+import socket
+import sys
+# pylint: disable=import-error
+from impala.dbapi import connect
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+logging.basicConfig()
+log = logging.getLogger(os.path.basename(sys.argv[0]))
+
+host_envs = [
+ 'IMPALA_HOST',
+ 'HOST'
+]
+
+port_envs = [
+ 'IMPALA_PORT',
+ 'PORT'
+]
+
+def getenvs(keys, default=None):
+ for key in keys:
+ value = os.getenv(key)
+ if value:
+ return value
+ return default
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Computes statistics on all Impala tables/partitions matching database / table / partition regexes")
+ parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\
+ help='Impalad host ' + \
+ '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')')
+ parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050),
+ help='Impalad port (default: 21050, ' + ', $'.join(port_envs) + ')')
+ parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)')
+ parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)')
+ parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)')
+ parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)')
+ parser.add_argument('-n', '--krb5-service-name', default='hive',
+ help='Service principal (default: \'hive\')')
+ parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL')
+ parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode')
+ args = parser.parse_args()
+
+ if args.verbose:
+ log.setLevel(logging.INFO)
+
+ return args
+
+def connect_db(args, database):
+ auth_mechanism = None
+ if args.kerberos:
+ auth_mechanism = 'GSSAPI'
+
+ log.info('connecting to %s:%s database %s', args.host, args.port, database)
+ return connect(
+ host=args.host,
+ port=args.port,
+ auth_mechanism=auth_mechanism,
+ use_ssl=args.ssl,
+ #user=user,
+ #password=password,
+ database=database,
+ kerberos_service_name=args.krb5_service_name
+ )
+
+def main():
+ args = parse_args()
+
+ try:
+ database_regex = re.compile(args.database, re.I)
+ table_regex = re.compile(args.table, re.I)
+ partition_regex = re.compile(args.partition, re.I)
+ except re.error as _:
+ log.error('error in provided regex: %s', _)
+ sys.exit(3)
+
+ conn = connect_db(args, None)
+
+ log.info('querying databases')
+ with conn.cursor() as db_cursor:
+ db_cursor.execute('show databases')
+ for db_row in db_cursor:
+ database = db_row[0]
+ if not database_regex.search(database):
+ log.debug("skipping database '%s', does not match regex '%s'", database, args.database)
+ continue
+ log.info('querying tables for database %s', database)
+ #db_conn = connect_db(args, database)
+ #with db_conn.cursor() as table_cursor:
+ with conn.cursor() as table_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ #table_cursor.execute('use %(database)s', {'database': database})
+ table_cursor.execute('use {}'.format(database))
+ table_cursor.execute('show tables')
+ for table_row in table_cursor:
+ table = table_row[0]
+ if not table_regex.search(table):
+ log.debug("skipping database '%s' table '%s', does not match regex '%s'", \
+ database, table, args.table)
+ continue
+ compute_table_stats(conn, args, database, table, partition_regex)
+
+def compute_table_stats(conn, args, database, table, partition_regex):
+ log.info("getting partitions for database '%s' table '%s'", database, table)
+ partitions_found = False
+ with conn.cursor() as partition_cursor:
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('use {}'.format(database))
+ partition_cursor.execute('show partitions {}'.format(database))
+ for partitions_row in partition_cursor:
+ partition_key = partitions_row[0]
+ partition_value = partitions_row[1]
+ partitions_found = True
+ if not partition_regex.match(partition_value):
+ # pylint: disable=logging-not-lazy
+ log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " +
+ "value does not match regex '%s'",
+ database,
+ table,
+ partition_key,
+ partition_value,
+ args.partition)
+ continue
+ # doesn't support parameterized query quoting from dbapi spec
+ partition_cursor.execute('COMPUTE INCREMENTAL STATS {db}.{table} PARTITION({key}={value})'\
+ .format(db=database, table=table, key=partition_key, value=partition_value))
+ if not partitions_found:
+ log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table)
+ with conn.cursor() as table_cursor:
+ log.info("running compute stats on table '%s'", table)
+ # doesn't support parameterized query quoting from dbapi spec
+ table_cursor.execute('COMPUTE STATS {db}.{table}'.format(db=database, table=table))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/impala_foreach_table.py b/impala_foreach_table.py
new file mode 100755
index 000000000..f4e324ac9
--- /dev/null
+++ b/impala_foreach_table.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and execute a query for each table in each database,
+or only those matching given db / table regexes
+
+Useful for getting row counts of all tables or analyzing tables:
+
+eg.
+
+impala_foreach_table.py --query 'SELECT COUNT(*) FROM {db}.{table}'
+impala_foreach_table.py --query 'COMPUTE STATS {table}'
+
+or just for today's partition:
+
+impala_foreach_table.py --query "COMPUTE INCREMENTAL STATS {db}.{table} PARTITION(date=$(date '+%Y-%m-%d'))"
+
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_foreach_table import HiveForEachTable
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaForEachTable(HiveForEachTable):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaForEachTable, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaForEachTable().main()
diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py
new file mode 100755
index 000000000..0ac8491dd
--- /dev/null
+++ b/impala_schemas_csv.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and dump all the schemas, tables and columns out in CSV format to stdout
+
+In practice Hive is much more reliable than Impala for dumping masses of schema (see adjacent hive_schemas_csv.py)
+
+Impala appears faster initially but then slows down more than Hive and hits things query handle errors
+under sustained load of extracting large amounts of schema information
+
+There is also a risk that Impala's metadata may be out of date, so Hive is strongly preferred for this
+
+
+CSV format:
+
+database,table,column,type
+
+
+I recommend generating quoted csv because you may encounter Hive data types such as decimal(15,2)
+which would cause incorrect field splitting, you can disable by setting --quotechar='' to blank but
+if escaping is needed then you will be forced to specify an --escapechar otherwise the csv writer will
+raise a traceback to tell you to set one (eg. --escapechar='\\')
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+from hive_schemas_csv import HiveSchemasCSV
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class ImpalaSchemasCSV(HiveSchemasCSV):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaSchemasCSV, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaSchemasCSV().main()
diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py
new file mode 100755
index 000000000..e62027542
--- /dev/null
+++ b/impala_tables_column_counts.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and count the number of columns for each table in each database,
+or only those matching given db / table regexes
+
+You can also get this from the schemas.csv output generated by impala_schemas_csv.py, eg.
+
+ tail -n +2 impala_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_column_counts import HiveTablesColumnCounts
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesColumnCounts(HiveTablesColumnCounts):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesColumnCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesColumnCounts().main()
diff --git a/impala_tables_list.py b/impala_tables_list.py
new file mode 100755
index 000000000..a45194ad3
--- /dev/null
+++ b/impala_tables_list.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and list all databases and tables
+
+TSV Output format:
+
+
+
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_list import HiveTablesList
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+
+class ImpalaTablesList(HiveTablesList):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesList, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesList().main()
diff --git a/impala_tables_locations.py b/impala_tables_locations.py
new file mode 100755
index 000000000..383bb1154
--- /dev/null
+++ b/impala_tables_locations.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and list the locations of all tables in all databases,
+or only those matching given db / table regexes
+
+Caveats:
+
+ Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't
+
+ Impala is faster than Hive for the first ~1000 tables but then slows down
+ so if you have a lot of tables I recommend you use the Hive version of this instead
+ eg. by ~1900 tables the Hive version will overtake the Impala version and
+ for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_locations import HiveTablesLocations
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesLocations(HiveTablesLocations):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesLocations, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesLocations().main()
diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py
new file mode 100755
index 000000000..501b82c93
--- /dev/null
+++ b/impala_tables_metadata.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and print the first matching DDL metadata field (eg. 'Location')
+for each table in each database, or only those matching given db / table regexes
+
+Examples (fields are case sensitive regex and return N/A without match):
+
+./impala_tables_metadata.py --field Location ...
+./impala_tables_metadata.py --field SerDe ...
+
+Caveats:
+
+ Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't
+
+ Impala is faster than Hive for the first ~1000 tables but then slows down
+ so if you have a lot of tables I recommend you use the Hive version of this instead
+ eg. by ~1900 tables the Hive version will overtake the Impala version and
+ for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_metadata import HiveTablesMetadata
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesMetadata(HiveTablesMetadata):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesMetadata, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesMetadata().main()
diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py
new file mode 100755
index 000000000..2b70caf3f
--- /dev/null
+++ b/impala_tables_null_columns.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and find tables with columns containing only NULLs
+for all tables in all databases, or only those matching given db / table regexes
+
+Describes each table, constructs a complex query to check each column individually for containing only NULLs,
+and prints out each tables' count of total columns containing only NULLs as well as the list of offending columns
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2014 from my DevOps Perl Tools repo
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_null_columns import HiveTablesNullColumns
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesNullColumns(HiveTablesNullColumns):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesNullColumns, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesNullColumns().main()
diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py
new file mode 100755
index 000000000..4fea8a82e
--- /dev/null
+++ b/impala_tables_null_rows.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and count number of rows with only NULLs in all columns
+for each tables in each database, or only those matching given db / table regexes
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_null_rows import HiveTablesNullRows
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesNullRows(HiveTablesNullRows):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesNullRows, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesNullRows().main()
diff --git a/impala_tables_row_column_counts.py b/impala_tables_row_column_counts.py
new file mode 100755
index 000000000..8cf4d6b82
--- /dev/null
+++ b/impala_tables_row_column_counts.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and count the number of rows and columns for each table
+in each database, or only those matching given db / table regexes
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_row_column_counts import HiveTablesRowColumnCounts
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesRowColumnCounts(HiveTablesRowColumnCounts):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesRowColumnCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesRowColumnCounts().main()
diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py
new file mode 100755
index 000000000..decc2982c
--- /dev/null
+++ b/impala_tables_row_counts.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and get rows counts for all tables in all databases,
+or only those matching given db / table / partition value regexes
+
+Useful for reconciliations between clusters after migrations
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+lib = os.path.join(srcdir, 'lib')
+sys.path.append(pylib)
+sys.path.append(lib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_row_counts import HiveTablesRowCounts
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class ImpalaTablesRowCounts(HiveTablesRowCounts):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesRowCounts, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesRowCounts().main()
diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py
new file mode 100755
index 000000000..d7b49a3b2
--- /dev/null
+++ b/impala_tables_row_counts_any_nulls.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Connect to an Impala daemon and count number of rows with NULL in any column
+for each table in each database, or only those matching given db / table regexes
+
+Useful for catching problems with data quality or subtle ETL bugs
+
+Rewrite of a Perl version from 2013 from my DevOps Perl Tools repo
+
+Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
+
+Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see
+
+https://github.com/cloudera/impyla/issues/286
+
+If you get an error like this:
+
+ERROR:impala.hiveserver2:Failed to open transport (tries_left=1)
+...
+TTransportException: TSocket read 0 bytes
+
+then check your --kerberos and --ssl settings match the cluster's settings
+(Thrift and Kerberos have the worst error messages ever)
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import sys
+srcdir = os.path.abspath(os.path.dirname(__file__))
+pylib = os.path.join(srcdir, 'pylib')
+sys.path.append(pylib)
+try:
+ # pylint: disable=wrong-import-position
+ from hive_tables_row_counts_any_nulls import HiveTablesRowsWithNulls
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.4.0'
+
+
+class ImpalaTablesRowsWithNulls(HiveTablesRowsWithNulls):
+
+ def __init__(self):
+ # Python 2.x
+ super(ImpalaTablesRowsWithNulls, self).__init__()
+ # Python 3.x
+ # super().__init__()
+
+ # these are auto-set checking sys.argv[0] in HiveImpalaCLI class
+ self.name = 'Impala'
+ #self.default_port = 21050
+ #self.default_service_name = 'impala'
+
+
+if __name__ == '__main__':
+ ImpalaTablesRowsWithNulls().main()
diff --git a/ipython_notebook_pyspark.py b/ipython_notebook_pyspark.py
index 013d61ff2..4c4505015 100755
--- a/ipython_notebook_pyspark.py
+++ b/ipython_notebook_pyspark.py
@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
#
# Author: Hari Sekhon
# Date: 6/8/2014
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -93,7 +93,7 @@
# TODO: rewrite from here as a CLI class, not using globals
if len(sys.argv) > 1:
- printerr("""Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+ printerr("""Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
usage: %s
diff --git a/json_docs_to_bulk_multiline.py b/json_docs_to_bulk_multiline.py
index 98f4d4db9..eb49bcbeb 100755
--- a/json_docs_to_bulk_multiline.py
+++ b/json_docs_to_bulk_multiline.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2017-07-28 17:08:47 +0200 (Fri, 28 Jul 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/json_to_xml.py b/json_to_xml.py
index ca2afcafb..54c0d6395 100755
--- a/json_to_xml.py
+++ b/json_to_xml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-15 00:07:09 +0000 (Fri, 15 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,7 +52,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.1'
+__version__ = '0.2.0'
class JsonToXml(CLI):
@@ -110,7 +110,7 @@ def process_file(self, filepath):
if filepath == '-':
filepath = ''
if filepath == '':
- self.json_to_xml(sys.stdin.read())
+ print(self.json_to_xml(sys.stdin.read()))
else:
with open(filepath) as _:
content = _.read()
diff --git a/json_to_yaml.py b/json_to_yaml.py
new file mode 100755
index 000000000..674f6729f
--- /dev/null
+++ b/json_to_yaml.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-12-19 17:54:21 +0000 (Thu, 19 Dec 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tool to convert JSON to YAML
+
+Reads any given files as JSON and prints the equivalent YAML to stdout for piping or redirecting to a file.
+
+Directories if given are detected and recursed, processing all files in the directory tree ending in a .json suffix.
+
+Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from
+standard input.
+
+Written to convert old AWS CloudFormation json templates to yaml
+
+See also:
+
+
+ json2yaml.sh - https://github.com/HariSekhon/DevOps-Bash-tools
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import json
+import os
+import re
+import sys
+import yaml
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import die, ERRORS, log, log_option
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.2.0'
+
+
+class JsonToYaml(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(JsonToYaml, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.re_json_suffix = re.compile(r'.*\.json$', re.I)
+
+ @staticmethod
+ def json_to_yaml(content, filepath=None):
+ try:
+ _ = json.loads(content)
+ except (KeyError, ValueError) as _:
+ file_detail = ''
+ if filepath is not None:
+ file_detail = ' in file \'{0}\''.format(filepath)
+ die("Failed to parse JSON{0}: {1}".format(file_detail, _))
+ return yaml.safe_dump(_)
+
+ def run(self):
+ if not self.args:
+ self.args.append('-')
+ for arg in self.args:
+ if arg == '-':
+ continue
+ if not os.path.exists(arg):
+ print("'%s' not found" % arg)
+ sys.exit(ERRORS['WARNING'])
+ if os.path.isfile(arg):
+ log_option('file', arg)
+ elif os.path.isdir(arg):
+ log_option('directory', arg)
+ else:
+ die("path '%s' could not be determined as either a file or directory" % arg)
+ for arg in self.args:
+ self.process_path(arg)
+
+ def process_path(self, path):
+ if path == '-' or os.path.isfile(path):
+ self.process_file(path)
+ elif os.path.isdir(path):
+ for root, _, files in os.walk(path):
+ for filename in files:
+ filepath = os.path.join(root, filename)
+ if self.re_json_suffix.match(filepath):
+ self.process_file(filepath)
+ else:
+ die("failed to determine if path '%s' is a file or directory" % path)
+
+ def process_file(self, filepath):
+ log.debug('processing filepath \'%s\'', filepath)
+ if filepath == '-':
+ filepath = ''
+ if filepath == '':
+ print(self.json_to_yaml(sys.stdin.read()))
+ else:
+ with open(filepath) as _:
+ content = _.read()
+ print('---')
+ print(self.json_to_yaml(content, filepath=filepath))
+
+
+if __name__ == '__main__':
+ JsonToYaml().main()
diff --git a/jython_autoinstall.exp b/jython_autoinstall.exp
index 1230a03a1..342488e6a 100755
--- a/jython_autoinstall.exp
+++ b/jython_autoinstall.exp
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: Fri Jun 17 15:12:17 2016 +0100
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set force_conservative 0 ;# set to 1 to force conservative mode even if
diff --git a/jython_install.sh b/jython_install.sh
index 690b0421a..4d3f420ee 100755
--- a/jython_install.sh
+++ b/jython_install.sh
@@ -4,18 +4,17 @@
# Author: Hari Sekhon
# Date: 2016-08-01 10:17:55 +0100 (Mon, 01 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
[ -n "${DEBUG:-}" ] && set -x
-srcdir="$(cd "$(dirname "$0")" && pwd)"
JYTHON_VERSION="${1:-2.7.0}"
@@ -23,7 +22,7 @@ JYTHON_INSTALL_DIR="${2:-/opt/jython-$JYTHON_VERSION}"
# not set in busybox
#[ $EUID -eq 0 ] && sudo="" || sudo=sudo
-[ $(whoami) = "root" ] && sudo="" || sudo=sudo
+[ "$(whoami)" = "root" ] && sudo="" || sudo=sudo
# installer will tell us if dir isn't empty
#if ! [ -e "$JYTHON_INSTALL_DIR" ]; then
diff --git a/kics.config b/kics.config
new file mode 100644
index 000000000..51ac5712e
--- /dev/null
+++ b/kics.config
@@ -0,0 +1,44 @@
+#
+# Author: Hari Sekhon
+# Date: 2023-05-05 18:05:53 +0100 (Fri, 05 May 2023)
+#
+# vim:ts=2:sts=2:sw=2:et:filetype=yaml
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# K i c s C o n f i g
+# ============================================================================ #
+
+# https://github.com/Checkmarx/kics/blob/master/docs/configuration-file.md
+
+---
+#path: assets/iac_samples
+verbose: true
+log-file: true
+#type:
+# - Dockerfile
+# - Kubernetes
+#queries-path: "assets/queries"
+exclude-paths:
+ # ignore submodules - handle them in the source repos only
+ - bash-tools/
+ - github-actions/
+ - haproxy-configs/
+ - jenkins/
+ - kubernetes-templates/
+ - lib/
+ - pylib/
+ - spotify-tools/
+ - sql/
+ - sql-keywords/
+ - templates/
+ - terraform-templates/
+#output-path: "results"
diff --git a/lib/__init__.py b/lib/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/hive_impala_cli.py b/lib/hive_impala_cli.py
new file mode 100755
index 000000000..958ccec69
--- /dev/null
+++ b/lib/hive_impala_cli.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import socket
+import sys
+from impala.dbapi import connect
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_host, validate_port
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.5.0'
+
+
+class HiveImpalaCLI(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(HiveImpalaCLI, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.name = ['HiveServer2', 'Hive']
+ self.host = None
+ self.port = None
+ self.default_host = socket.getfqdn()
+ self.default_port = 10000
+ self.default_service_name = 'hive'
+ self.kerberos = False
+ self.krb5_service_name = self.default_service_name
+ self.ssl = False
+ self.verbose_default = 1
+ #self.timeout_default = 86400
+ self.timeout_default = None
+ if 'impala' in sys.argv[0]:
+ self.name = 'Impala'
+ self.default_port = 21050
+ self.default_service_name = 'impala'
+
+ def add_options(self):
+ super(HiveImpalaCLI, self).add_options()
+ self.add_hostoption()
+ self.add_opt('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)')
+ self.add_opt('-n', '--krb5-service-name', default=self.default_service_name,
+ help='Service principal (default: {})'.format(self.default_service_name))
+ self.add_opt('-S', '--ssl', action='store_true', help='Use SSL')
+
+ def process_options(self):
+ super(HiveImpalaCLI, self).process_options()
+ self.host = self.get_opt('host')
+ self.port = self.get_opt('port')
+ validate_host(self.host)
+ validate_port(self.port)
+ self.port = int(self.port)
+ self.kerberos = self.get_opt('kerberos')
+ self.krb5_service_name = self.get_opt('krb5_service_name')
+ self.ssl = self.get_opt('ssl')
+
+ def connect(self, database):
+ auth_mechanism = None
+ if self.kerberos:
+ auth_mechanism = 'GSSAPI'
+ log.debug('kerberos enabled')
+ log.debug('krb5 remote service principal name = %s', self.krb5_service_name)
+ if self.ssl:
+ log.debug('ssl enabled')
+
+ log.info('connecting to %s:%s database %s', self.host, self.port, database)
+ return connect(
+ host=self.host,
+ port=self.port,
+ auth_mechanism=auth_mechanism,
+ use_ssl=self.ssl,
+ #user=user,
+ #password=password,
+ database=database,
+ kerberos_service_name=self.krb5_service_name
+ )
diff --git a/lib/postgres_cli.py b/lib/postgres_cli.py
new file mode 100755
index 000000000..e84416d98
--- /dev/null
+++ b/lib/postgres_cli.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-12 17:39:57 +0000 (Thu, 12 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import socket
+import sys
+import psycopg2
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log, validate_host, validate_port, validate_user, validate_password
+ from harisekhon import CLI
+except ImportError as _:
+ print('module import failed: %s' % _, file=sys.stderr)
+ print("Did you remember to build the project by running 'make'?", file=sys.stderr)
+ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+
+class PostgreSQLCLI(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(PostgreSQLCLI, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.name = ['PostgreSQL', 'Postgres', 'PG']
+ self.host = None
+ self.port = None
+ self.default_host = socket.getfqdn()
+ self.default_port = 5432
+ self.user = None
+ self.password = None
+ #self.ssl = False
+ self.verbose_default = 1
+ self.timeout_default = None
+
+ def add_options(self):
+ super(PostgreSQLCLI, self).add_options()
+ self.add_hostoption()
+ self.add_useroption()
+ self.add_opt('-d', '--database', help='Database to connect to')
+
+ def process_options(self):
+ super(PostgreSQLCLI, self).process_options()
+ self.host = self.get_opt('host')
+ self.host = self.get_opt('host')
+ self.user = self.get_opt('user')
+ self.password = self.get_opt('password')
+ validate_host(self.host)
+ validate_port(self.port)
+ validate_user(self.user)
+ validate_password(self.password)
+ self.port = int(self.port)
+ #self.ssl = self.get_opt('ssl')
+
+ def connect(self, database):
+ log.info('connecting to %s:%s database %s as user %s', self.host, self.port, database)
+ return psycopg2.connect(host=self.host,
+ port=self.port,
+ database=database,
+ user=self.user,
+ password=self.password)
diff --git a/opentsdb_import_metric_distribution.py b/opentsdb_import_metric_distribution.py
index 1331b68d2..3b3d3df09 100755
--- a/opentsdb_import_metric_distribution.py
+++ b/opentsdb_import_metric_distribution.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-10 11:47:12 +0100 (Mon, 10 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/opentsdb_list_metrics.sh b/opentsdb_list_metrics.sh
index b82c66113..a119bd9fa 100755
--- a/opentsdb_list_metrics.sh
+++ b/opentsdb_list_metrics.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-13 22:36:14 +0100 (Fri, 13 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -72,7 +72,7 @@ done
check_bin(){
local bin="$1"
- if ! which $bin &>/dev/null; then
+ if ! type -P "$bin" &>/dev/null; then
echo "'$bin' command not found in \$PATH ($PATH)"
exit 1
fi
@@ -83,6 +83,8 @@ check_bin jq
if [ -z "${DEBUG:-}" ]; then
curl_options="$curl_options -s"
fi
+# split opts
+# shellcheck disable=SC2086
curl $curl_options "$tsd_url/api/suggest?type=$metrics&q=&max=2000000000" |
jq '.[]' |
sed 's/"//g' |
diff --git a/opentsdb_list_metrics_hbase.sh b/opentsdb_list_metrics_hbase.sh
index 4b2781d70..86142e4c0 100755
--- a/opentsdb_list_metrics_hbase.sh
+++ b/opentsdb_list_metrics_hbase.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-13 22:36:14 +0100 (Fri, 13 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -73,7 +73,7 @@ done
check_bin(){
local bin="$1"
- if ! which $bin &>/dev/null; then
+ if ! type -P "$bin" &>/dev/null; then
echo "'$bin' command not found in \$PATH ($PATH)"
exit 1
fi
@@ -91,6 +91,7 @@ print_metrics(){
print_metrics_by_age(){
# hackish but convenient - forking to the date command thousands or hundreds of thousands of times can take hours, python takes 10 secs even for 250,000+ metrics
tmp_python_script=$(mktemp)
+ # shellcheck disable=SC1117
cat > "$tmp_python_script" < 'name:$metrics', VERSIONS => 1 }" 2>/dev/null |
diff --git a/pig_udfs.jy b/pig_udfs.jy
index 845a1f531..571e2cedc 100755
--- a/pig_udfs.jy
+++ b/pig_udfs.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2015-03-12 21:16:54 +0000 (Thu, 12 Mar 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/plot_uk_marriage_rates.py b/plot_uk_marriage_rates.py
new file mode 100755
index 000000000..a38332443
--- /dev/null
+++ b/plot_uk_marriage_rates.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2025-04-26 00:37:28 +0800 (Sat, 26 Apr 2025)
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Plots the Marriage rates in England & Wales using the latest download from the UK Government website:
+
+ https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/marriagecohabitationandcivilpartnerships/bulletins/marriagesinenglandandwalesprovisional/2021and2022
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import platform
+import subprocess
+import sys
+#import time
+import traceback
+import pandas as pd
+import matplotlib.pyplot as plt
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon.utils import log
+ from harisekhon import CLI
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1'
+
+
+# pylint: disable=too-few-public-methods
+class PlotUKMarriageRates(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(PlotUKMarriageRates, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.timeout_default = 0
+
+ # def add_options(self):
+ # super(PlotUKMarriageRates, self).add_options()
+ #
+ # def process_options(self):
+ # super(PlotUKMarriageRates, self).process_options()
+
+ def run(self):
+ file_path = f"{srcdir}/tests/data/uk_marriage_rates_2022.xslx"
+ if self.args:
+ file_path = self.args[0]
+ #self.usage("Provide path to datadownload.xlsx")
+
+ if not os.path.isfile(file_path):
+ self.usage(f"Invalid argument provided, not a file: {file_path}")
+
+ log.info(f"Loading file: {file_path}")
+ xls = pd.ExcelFile(file_path)
+
+ log.info("Parsing xls")
+ # Read the relevant sheet and skip the metadata
+ df = xls.parse('Figure 2', skiprows=6)
+
+ # Set proper column headers
+ df.columns = df.iloc[1]
+ df = df[2:] # Drop the header rows
+
+ # Rename columns for clarity
+ df.columns = ['Year', 'Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women']
+
+ # Convert data types
+ df['Year'] = df['Year'].astype(int)
+ for col in ['Opposite-sex Men', 'Opposite-sex Women', 'Same-sex Men', 'Same-sex Women']:
+ df[col] = pd.to_numeric(df[col], errors='coerce')
+
+ log.info("Plotting")
+ plt.figure(figsize=(10, 6))
+ plt.plot(df['Year'], df['Opposite-sex Men'], label='Opposite-sex Men')
+ plt.plot(df['Year'], df['Opposite-sex Women'], label='Opposite-sex Women')
+ plt.plot(df['Year'], df['Same-sex Men'], label='Same-sex Men')
+ plt.plot(df['Year'], df['Same-sex Women'], label='Same-sex Women')
+
+ plt.title('Marriage Rates Over Time (England & Wales)')
+ plt.xlabel('Year')
+ plt.ylabel('Marriage Rate per 1,000 People')
+ plt.legend()
+ plt.grid(True)
+ plt.tight_layout()
+ #plt.show()
+ #
+ # doesn't show anything because the script finishes before the GUI event loop
+ # has time to process and display the window
+ #plt.show(block=False)
+ #
+ # pylint: disable=line-too-long
+ #
+ # doesn't work even with this hack:
+ #
+ # WARNING: NSWindow geometry should only be modified on the main thread! This will raise an exception in the future
+ #
+ #import threading
+ #threading.Thread(target=plt.show).start()
+ #sleep_secs = 20
+ #log.info(f"Sleeping for {sleep_secs} secs to allow you to see the graph pop-up")
+ #time.sleep(sleep_secs)
+
+ image_path = os.path.splitext(file_path)[0] + '.png'
+ log.info("Generating output image: {image_path}")
+
+ #if os.path.exists(image_path):
+ # log.warning(f"Image page already exists, skipping recreating for safety: {image_path}")
+ # #plt.close()
+ # return
+
+ # doesn't solve blank png
+ #plt.gcf().canvas.draw()
+
+ # results in blank png
+ #plt.savefig(image_path)
+
+ fig = plt.gcf() # draw before saving
+ fig.canvas.draw() # force rendering
+ plt.savefig(image_path)
+
+ plt.show()
+
+ if platform.system() == "Darwin":
+ # fire and forget
+ # pylint: disable=subprocess-run-check
+ subprocess.run(['open', image_path])
+
+ #plt.close()
+
+
+if __name__ == '__main__':
+ PlotUKMarriageRates().main()
diff --git a/pylib b/pylib
index 1c4746337..0d272f4cf 160000
--- a/pylib
+++ b/pylib
@@ -1 +1 @@
-Subproject commit 1c4746337785c416fb535d16d5ec1f8455c96742
+Subproject commit 0d272f4cf2c27a8ee614eea18c0bd171e47a06cd
diff --git a/python_find_library_path.py b/python_find_library_path.py
new file mode 100755
index 000000000..66fb9c00e
--- /dev/null
+++ b/python_find_library_path.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-09-27
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Simple tool to print the local path to one or more libraries given as arguments
+
+Useful for finding where things are installed on different operating systems like Mac vs Linux
+
+If no libraries are specified, finds the default library location as if the 'sys' argument was given
+
+Tested on Python 2.7 and Python 3.x on Mac and Linux
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import getpass
+import os
+import sys
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3.0'
+
+libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+sys.path.append(libdir)
+
+
+def usage():
+ print(__doc__.strip())
+ print('\n\nusage: {} [ ...]\n'\
+ .format(os.path.basename(sys.argv[0])), file=sys.stderr)
+ sys.exit(3)
+
+def get_module_location(module):
+ # pylint: disable=wrong-import-position,import-error
+ imported_module = __import__(module)
+ # sys doesn't have __file__ but it does have base_prefix
+ attribs = dir(imported_module)
+ # - looks like 'sys.base_prefix' / 'sys.base_exec_prefix' are only available in IPython
+ # - don't use 'executable' as it will point to a generic location eg. /usr/bin/python
+ # or /usr/local/opt/python/bin/python3.7) instead of the actual library path
+ # like /usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7
+ #potential_attribs = ['__file__', 'exec_prefix', 'base_prefix', 'base_exec_prefix']
+ potential_attribs = ['__file__', 'exec_prefix'] #, 'executable']
+ for potential_attrib in potential_attribs:
+ if potential_attrib in attribs:
+ value = getattr(imported_module, potential_attrib)
+ if potential_attrib == 'exec_prefix':
+ if not 'Python.framework' in value and not 'site-packages' in value:
+ continue
+ return value
+ # to account for sys not having __file__ attribute
+ if '__name__' in attribs and imported_module.__name__ == 'sys':
+ user = getpass.getuser()
+ for path in sys.path:
+ # don't return local /Users/$USER/Library/Python/2.7/lib/python/site-packages
+ # as that is not the source of sys
+ if user in path:
+ continue
+ if path.endswith('/site-packages'):
+ #value = path.rsplit('/site-packages')[0]
+ return value
+ raise AttributeError('could not find any attribute to determine location out of: {}'\
+ .format(', '.join(potential_attribs)))
+
+def main():
+ exitcode = 0
+ if len(sys.argv) < 2:
+ sys.argv.append('sys')
+ for arg in sys.argv:
+ if arg.startswith('-'):
+ usage()
+ for module in sys.argv[1:]:
+ try:
+ print(get_module_location(module))
+ except AttributeError as _:
+ print('attributes not found in module: {} - {}'.format(module, _), file=sys.stderr)
+ exitcode = max(1, exitcode)
+ except ImportError as _:
+ print('module not found: {} - {}'.format(module, _), file=sys.stderr)
+ exitcode = max(2, exitcode)
+ sys.exit(exitcode)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/pythonpath.py b/pythonpath.py
new file mode 100755
index 000000000..ef42891b9
--- /dev/null
+++ b/pythonpath.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-09-27
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Simple tool to print the Python sys.path, one per line
+
+Tested on Python 2.7 and Python 3.x on Mac and Linux
+
+"""
+
+from __future__ import print_function
+#from __future__ import unicode_literals
+
+import sys
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.1.0'
+
+#libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib'))
+#sys.path.append(libdir)
+
+
+def main():
+ for path in sys.path:
+ if path:
+ print(path)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/quay_show_tags.py b/quay_show_tags.py
index 5743be955..d1eef86bf 100755
--- a/quay_show_tags.py
+++ b/quay_show_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -30,6 +30,7 @@
#from __future__ import unicode_literals
import os
+import re
import sys
import traceback
srcdir = os.path.abspath(os.path.dirname(__file__))
@@ -43,7 +44,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.6.2'
+__version__ = '0.6.3'
class QuayTags(DockerHubTags):
@@ -61,7 +62,10 @@ def run(self):
self.quiet = self.get_opt('quiet')
if not self.quiet:
print('\nQuay.io ', end='')
+ re_quay = re.compile('^quay.io/', re.I)
for arg in self.args:
+ if re_quay.match(arg):
+ arg = re_quay.sub('', arg)
self.print_tags(arg)
diff --git a/requirements.txt b/requirements.txt
index 9ded348a5..05d204c50 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,49 +1,97 @@
-avro>=1.8.1
-bitarray>=0.8.1
-#cassandra-driver>=3.6.0
-dicttoxml>=1.7.4
-#docker-py>=1.9.0
+avro==1.8.1
+
+# requires Python == 3.4, build in Makefile instead
+#avro-python3==1.9.0
+
+# AWS CLIv1 is obsolete and doesn't support SSO - use CLIv2 - see https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/install/install_aws_cli.sh
+#awscli==1.16.241
+#bitarray==0.8.1
+
+#boto==2.49.0
+boto3==1.10.37
+#cassandra-driver==3.6.0
+dicttoxml==1.7.4
+
# Elasticsearch library must match major version :(
# for Elasticsearch 2.x
#elasticsearch>=2.0.0,<3.0.0
# for Elasticsearch 1.x
#elasticsearch>=1.0.0,<2.0.0
+
# fails on requiring newer version of setuptools
-Flask>=0.10.1
-GitPython>=2.0.8
-happybase>=1.0.0
-humanize>=0.5.1
-#impyla>=0.12.0
-Jinja2>=2.8
-#kazoo>=2.2.1
-ldif3>=3.2.2
-linecache2>=1.0.0
-MarkupSafe>=0.23
-#Markdown>=2.6.8
-nose>=1.3.7
-numpy>=1.8.0rc1
-psutil>=4.3.0
-py>=1.4.31
+#Flask==0.10.1
+GitPython==2.1.15
+
+# this GCP API is surprisingly awful, not using
+#google-api-python-client==1.11.0
+
+# XXX: install broken on new M1 Mac Python 3.9
+#happybase==1.0.0
+
+humanize==0.5.1
+impyla==0.19.0
+jinja2==2.11.3
+#kazoo==2.2.1
+ldif3==3.2.2
+#MarkupSafe==0.23
+#Markdown==2.6.8
+
+matplotlib==3.7.5
+
+# Python 3.5+
+#numpy==1.17.2
+# XXX: install broken on new M1 Mac Python 3.9
+#numpy==1.16.5
+
+# for plot_uk_marriage_rates.py
+openpyxl==3.1.5
+pandas==2.0.3
+
+# requires pg_config to build from source
+#psycopg2==2.8.4
+# XXX: install broken on new M1 Mac Python 3.9
+#psycopg2-binary==2.8.4
+
+python-cson==1.0.9
+psutil==5.7.0
+
# parquet support in pyarrow is weaker, gone back to using parquet-tools
#pyarrow==0.6.0
-#python-ldap>=2.4.41
-# PyLint fails some checks in Travis that pass locally
-#pylint==1.5.2
-#pyhs2>=0.6.0
-#pytest>=3.0.0
-#python-jenkins>=0.4.13
-#python-krbV>=1.0.90
+#PyHive==0.6.1
+
+# doesn't work with non-trivial code
+#PyInstaller==3.3.1
+
+# gcc compile error on Alpine, don't think this is used either
+#python-ldap==3.2.0
+
+#python-jenkins==0.4.13
+# pulled in automatically by snakebite[kerberos]
+#python-krbV==1.0.90
# needed by avro
-python-snappy>=0.5
-PyYAML>=3.11
-requests>=2.11.1
-sasl>=0.2.1
-sh>=1.12.14
-six>=1.10.0
-#snakebite[kerberos]>=2.11.0
-thrift-sasl>=0.2.1
-thrift>=0.9.3
-traceback2>=1.4.0
-unittest2>=1.1.0
-#wheel>=0.29.0
-xmltodict>=0.10.2
+
+# XXX: install broken on new M1 Mac Python 3.9
+#python-snappy==0.5
+
+# XXX: install broken on new M1 Mac Python 3.9
+#sasl==0.2.1
+
+sh==1.12.14
+selenium==3.141.0
+
+# pulls in python-KrbV as a dependency which doesn't build on Mac any more
+# relies on python-krbV is unmaintained and unported to Python 3
+# - moved to Makefile as best effort
+#snakebite[kerberos]==2.11.0
+#snakebite==2.11.0
+
+# XXX: install broken on new M1 Mac Python 3.9
+#thrift-sasl==0.2.1
+#thrift==0.9.3
+#thriftpy==0.3.9
+
+toml==0.10.0
+xmltodict==0.10.2
+yamllint==1.15.0
+
+#pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability. update: this breaks Python 3.5 build where this requirement is not found
diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py
new file mode 100755
index 000000000..c3d117e53
--- /dev/null
+++ b/selenium_hub_browser_test.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2021-05-12 09:55:01 +0100 (Wed, 12 May 2021)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn
+# and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+"""
+
+Tests a Selenium Hub / Selenoid using the given browsers eg. FIREFOX, CHROME
+against a given URL and content (defaults to google.com)
+
+Browsers default to 'FIREFOX' and 'CHROME' if not specified
+URL defaults to 'google.com' checking for content 'google'
+If you define a different URL then you must specify a --content or --regex validation otherwise none is used
+
+Example:
+
+ ./selenium_hub_browser_test.py --host [] []
+
+ ./selenium_hub_browser_test.py --hub-url http://:4444/wd/hub/ [] []
+
+Where browsers are one or more of these and must be supported by the remote Selenium Hub:
+
+ANDROID
+CHROME
+EDGE
+FIREFOX
+HTMLUNIT
+HTMLUNITWITHJS
+INTERNETEXPLORER
+IPAD
+IPHONE
+OPERA
+PHANTOMJS
+SAFARI
+WEBKITGTK
+
+Examples:
+
+ ./selenium_hub_browser_test.py --host x.x.x.x
+
+ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME
+
+ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google
+ ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*'
+
+
+Tested on Selenium Grid Hub v.3.141.59, v4.0.0 and Selenoid 1.10.1
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+import time
+import traceback
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+srcdir = os.path.abspath(os.path.dirname(__file__))
+libdir = os.path.join(srcdir, 'pylib')
+sys.path.append(libdir)
+try:
+ # pylint: disable=wrong-import-position
+ from harisekhon import CLI
+ from harisekhon.utils import log, validate_host, validate_port, validate_url, validate_regex, die
+except ImportError as _:
+ print(traceback.format_exc(), end='')
+ sys.exit(4)
+
+__author__ = 'Hari Sekhon'
+__version__ = '0.3'
+
+
+class SeleniumHubBrowserTest(CLI):
+
+ def __init__(self):
+ # Python 2.x
+ super(SeleniumHubBrowserTest, self).__init__()
+ # Python 3.x
+ # super().__init__()
+ self.host = None
+ self.port = None
+ self.protocol = 'http'
+ self.name = 'Selenium Hub'
+ self.path = 'wd/hub'
+ self.hub_url = None
+ self.url_default = 'http://google.com'
+ self.url = self.url_default
+ self.expected_content = None
+ self.expected_content_default = 'google'
+ self.expected_regex = None
+ self.timeout_default = 600
+ self.verbose_default = 2
+
+ def add_options(self):
+ super(SeleniumHubBrowserTest, self).add_options()
+ self.add_hostoption(name='Selenium Hub', default_port=4444)
+ self.add_opt('-U', '--hub-url', help='Selenium Hub URL (takes priority over --host/--port/--ssl)')
+ self.add_opt('-u', '--url', default=self.url_default,
+ help='URL to use for the test (default: {})'.format(self.url_default))
+ self.add_opt('-c', '--content', help='URL content to expect')
+ self.add_opt('-r', '--regex', help='URL content to expect')
+ self.add_opt('-S', '--ssl', action='store_true', help='Use SSL to connect to Selenium Hub')
+
+ def process_options(self):
+ super(SeleniumHubBrowserTest, self).process_options()
+ self.hub_url = self.get_opt('hub_url')
+ if self.hub_url:
+ validate_url(self.hub_url, 'hub')
+ else:
+ self.host = self.get_opt('host')
+ self.port = self.get_opt('port')
+ validate_host(self.host)
+ validate_port(self.port)
+ if self.get_opt('ssl') or int(self.port) == 443:
+ self.protocol = 'https'
+ self.hub_url = '{protocol}://{host}:{port}/{path}'\
+ .format(protocol=self.protocol, \
+ host=self.host, \
+ port=self.port, \
+ path=self.path)
+ self.url = self.get_opt('url')
+ if ':' not in self.url:
+ self.url = 'http://' + self.url
+ validate_url(self.url)
+ self.expected_content = self.get_opt('content')
+ self.expected_regex = self.get_opt('regex')
+ if self.expected_regex:
+ validate_regex(self.expected_regex)
+ self.expected_regex = re.compile(self.expected_regex)
+ elif self.url == self.url_default:
+ self.expected_content = self.expected_content_default
+ if not self.args:
+ # test basic Chrome and Firefox are available
+ self.args.append('chrome')
+ self.args.append('firefox')
+
+ def check_selenium(self, browser):
+ log.info("Connecting to '%s' for browser '%s'", self.hub_url, browser)
+ driver = webdriver.Remote(
+ command_executor=self.hub_url,
+ desired_capabilities=getattr(DesiredCapabilities, browser)
+ )
+ log.info("Checking url '%s'", self.url)
+ driver.get(self.url)
+ content = driver.page_source
+ title = driver.title
+ driver.quit()
+ if self.expected_regex:
+ log.info("Checking url content matches regex")
+ if not self.expected_regex.search(content):
+ die('ERROR: Page source content failed regex search')
+ elif self.expected_content:
+ log.info("Checking url content matches '%s'", self.expected_content)
+ if self.expected_content not in content:
+ die('ERROR: Page source content failed content match')
+ # not really recommended but in this case we cannot predict
+ # what to expect on a random url if not specified by --content/--regex (provided in the default test case)
+ #
+ # https://www.selenium.dev/documentation/en/worst_practices/http_response_codes/
+ elif '404' in title:
+ die('ERROR: Page title contains a 404 / error ' +
+ '(if this is expected, specify --content / --regex to check instead): {}'.format(title))
+ log.info("Succeeded for browser '%s' against url '%s'", browser, self.url)
+
+ def run(self):
+ start_time = time.time()
+ for browser in self.args:
+ self.check_selenium(browser.upper())
+ query_time = time.time() - start_time
+ log.info('Finished checks in {:.2f} secs'.format(query_time))
+
+
+if __name__ == '__main__':
+ SeleniumHubBrowserTest().main()
diff --git a/serf_event_handler.py b/serf_event_handler.py
index 38a0df1b4..c0d5cacbb 100755
--- a/serf_event_handler.py
+++ b/serf_event_handler.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-16 15:44:16 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -61,7 +61,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.2.2'
+__version__ = '0.2.3'
class SerfEventHandler(CLI):
@@ -95,7 +95,7 @@ def add_options(self):
def enable_commands(self):
if self.event in ['query', 'event']:
- cmd = None
+ cmd = ''
if self.event == 'query':
cmd = self.query_name
elif self.event == 'user':
diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt
old mode 100755
new mode 100644
index 980b0ea40..a8fe03ea8
--- a/setup/apk-packages-dev.txt
+++ b/setup/apk-packages-dev.txt
@@ -2,25 +2,20 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# Alpine Dev Package Requirements
# ============================================================================ #
-alpine-sdk
-gcc
-cyrus-sasl-dev
-krb5-dev
-libffi-dev
-linux-headers
-#openssl-dev
-libressl-dev
-python-dev
-wget
-zip
+openldap-dev
+postgresql-dev
+snappy-dev
+
+# installed by bash-tools submodule now
+#unzip
diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt
new file mode 100644
index 000000000..1b9ee9d77
--- /dev/null
+++ b/setup/apk-packages-pip.txt
@@ -0,0 +1,23 @@
+#
+# Author: Hari Sekhon
+# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying LICENSE file
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# Alpine PIP Package Requirements
+# ============================================================================ #
+
+py-dicttoxml
+py-psutil
+py-pyldap
+py3-jinja2
+py3-numpy
+#py3-flask
+py3-pygit2
+py3-psycopg2
diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt
old mode 100755
new mode 100644
index a06d0f6ab..267f430e5
--- a/setup/apk-packages.txt
+++ b/setup/apk-packages.txt
@@ -2,30 +2,17 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# Alpine Package Requirements
# ============================================================================ #
-bash
-# without this arguments to mktemp fail in json tests
-coreutils
-git
-grep
-make
-py2-pip
-python
-snappy-dev
-
-# needed for tests/test_spark* and ambari_cancel_all_requests.sh
-curl
-
# for htpasswd for docker registry authenticated checks
apache2-utils
@@ -33,3 +20,20 @@ apache2-utils
# Spark Java Py4J gets java linking error without this
#if [ -f /lib/libc.musl-x86_64.so.1 ]; then [ -e /lib/ld-linux-x86-64.so.2 ] || ln -sv /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2; fi
+
+# =====================================
+# installed by bash-tools submodule now
+
+# full mktemp needed for json tests
+# full split needed for anonymize_parallel.sh
+#coreutils
+
+# needed for tests/test_spark* and ambari_cancel_all_requests.sh
+#curl
+
+#grep
+
+# for anonymize_parallel.sh
+#parallel
+
+#zip
diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh
new file mode 100755
index 000000000..d7bedf92a
--- /dev/null
+++ b/setup/bootstrap.sh
@@ -0,0 +1,86 @@
+#!/bin/sh
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-10-16 10:33:03 +0100 (Wed, 16 Oct 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# Alpine / Wget:
+#
+# wget -O- https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh
+#
+# Curl:
+#
+# curl https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh
+
+set -eu
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(dirname "$0")"
+
+repo="https://github.com/HariSekhon/DevOps-Python-tools"
+
+directory="pytools"
+
+sudo=""
+[ "$(whoami)" = "root" ] || sudo=sudo
+
+if [ "$(uname -s)" = Darwin ]; then
+ echo "Bootstrapping on Mac OS X: $repo"
+ if ! type brew >/dev/null 2>&1; then
+ curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install | $sudo ruby
+ fi
+elif [ "$(uname -s)" = Linux ]; then
+ echo "Bootstrapping on Linux: $repo"
+ if type apk >/dev/null 2>&1; then
+ $sudo apk --no-cache add bash git make curl wget
+ elif type apt-get >/dev/null 2>&1; then
+ if [ -n "${CI:-}" ]; then
+ export DEBIAN_FRONTEND=noninteractive
+ fi
+ opts="-o DPkg::Lock::Timeout=1200"
+ if [ -z "${PS1:-}" ]; then
+ opts="$opts -qq"
+ fi
+ $sudo apt-get update $opts
+ $sudo apt-get install $opts -y git make curl wget --no-install-recommends
+ elif type yum >/dev/null 2>&1; then
+ if grep -qi 'NAME=.*CentOS' /etc/*release; then
+ echo "CentOS EOL detected, replacing yum base URL to vault to re-enable package installs"
+ $sudo sed -i 's/^[[:space:]]*mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Linux-*
+ $sudo sed -i 's|^#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|' /etc/yum.repos.d/CentOS-Linux-*
+ fi
+ $sudo yum install -y git make curl wget
+ else
+ echo "Package Manager not found on Linux, cannot bootstrap"
+ exit 1
+ fi
+else
+ echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time"
+ exit 1
+fi
+
+if [ "${srcdir##*/}" = setup ]; then
+ cd "$srcdir/.."
+else
+ # if this is an empty directory eg. a cache mount, then remove it to get a proper checkout
+ rmdir "$directory" 2>/dev/null || :
+ if [ -d "$directory" ]; then
+ cd "$directory"
+ git pull
+ else
+ git clone "$repo" "$directory"
+ cd "$directory"
+ fi
+fi
+
+if [ -z "${NO_MAKE:-}" ]; then
+ make
+fi
diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt
index abbbc3439..401e24b97 100644
--- a/setup/brew-packages.txt
+++ b/setup/brew-packages.txt
@@ -3,16 +3,24 @@
# Author: Hari Sekhon
# Date: 2018-05-22 11:15:16 +0100 (Tue, 22 May 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# Mac OS X - Homebrew Package Requirements
# ============================================================================ #
-openssl
-wget
+# installed by bash-tools submodule now
+# for anonymize_parallel.sh
+#parallel
+
+parquet-tools
+snappy
+yamllint
+
+# Mac already provides unzip - brew is keg only and doesn't link anyway to avoid clashes
+#unzip
diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh
new file mode 100755
index 000000000..b01a77d04
--- /dev/null
+++ b/setup/ci_bootstrap.sh
@@ -0,0 +1,94 @@
+#!/bin/sh
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2020-06-02 17:43:35 +0100 (Tue, 02 Jun 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# Designed to bootstrap all CI systems with retries to make sure the networking, package lists and package repos works before proceeding
+#
+# Minimizes CI build failures due to temporary networking blips, which happens more often than you would think when you have a large number of CI builds across a lot of disparate systems
+
+set -eu
+[ -n "${DEBUG:-}" ] && set -x
+
+max_tries=10
+interval=60 # secs
+
+sudo=""
+# EUID undefined in posix sh
+#[ $EUID = 0 ] || sudo=sudo
+[ "$(whoami)" = root ] || sudo=sudo
+
+retry(){
+ # no local in posix sh
+ count=0
+ while true; do
+ # no let or bare (()) in posix sh, must discard output rather than execute it
+ _=$((count+=1))
+ printf "%s try %d: " "$(date '+%F %T')" "$count"
+ echo "$*"
+ "$@" &&
+ break;
+ echo
+ if [ "$count" -ge "$max_tries" ]; then
+ echo "$count tries failed, aborting..."
+ exit 1
+ fi
+ echo "sleeping for $interval secs before retrying"
+ sleep "$interval"
+ echo
+ done
+}
+
+if [ "$(uname -s)" = Darwin ]; then
+ echo "Bootstrapping Mac"
+ # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make)
+ #retry "$srcdir/../install/install_homebrew.sh"
+ if command -v brew 2>&1; then
+ # fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds
+ for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do
+ if [ -d "$git_root" ]; then
+ # find out if Homebrew is a shallow git checkout and if so fix it
+ if [ -f "$(git -C "$git_root" rev-parse --git-dir)/shallow" ] ||
+ [ "$(git -C "$git_root" rev-parse --is-shallow-repository)" = true ]; then
+ git -C "$git_root" fetch --unshallow
+ fi
+ fi
+ done
+ retry brew update
+ fi
+elif [ "$(uname -s)" = Linux ]; then
+ echo "Bootstrapping Linux"
+ if type apk >/dev/null 2>&1; then
+ retry $sudo apk update
+ retry $sudo apk add --no-progress bash git make
+ elif type apt-get >/dev/null 2>&1; then
+ opts="-q -o DPkg::Lock::Timeout=1200"
+ retry $sudo apt-get update $opts
+ retry $sudo apt-get install $opts -y git make
+ elif type yum >/dev/null 2>&1; then
+ #retry $sudo yum makecache
+ retry $sudo yum install -qy git make
+ else
+ echo "Package Manager not found on Linux, cannot bootstrap"
+ exit 1
+ fi
+else
+ echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time"
+ exit 1
+fi
+
+#retry make init
+
+# not calling make because in some CI systems we call 'make ci' which includes retries but in others with more restrictive build minutes we only run 'make' for a single shot build
+#
+#make
diff --git a/setup/ci_git_set_dir_safe.sh b/setup/ci_git_set_dir_safe.sh
new file mode 100755
index 000000000..66a53903a
--- /dev/null
+++ b/setup/ci_git_set_dir_safe.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2022-08-03 20:07:09 +0100 (Wed, 03 Aug 2022)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# Necessary for some CI/CD systems like Azure DevOps Pipelines which have incorrect ownership on the git checkout dir triggering this error:
+#
+# fatal: detected dubious ownership in repository at '/code/sql'
+
+# standalone script without lib dependency so it can be called directly from bootstrapped CI before submodules, since that is the exact problem that needs to be solved to allow CI/CD systems with incorrect ownership of the checkout directory to be able to checkout the necessary git submodules
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+dir="${1:-$srcdir/..}"
+
+cd "$dir"
+
+echo "Setting directory as safe: $PWD"
+git config --global --add safe.directory "$PWD"
+
+while read -r submodule_dir; do
+ dir="$PWD/$submodule_dir"
+ echo "Setting directory as safe: $dir"
+ git config --global --add safe.directory "$dir"
+done < <(git submodule | awk '{print $2}')
+
+echo "Done"
diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt
old mode 100755
new mode 100644
index 7f98d1c39..015439134
--- a/setup/deb-packages-dev.txt
+++ b/setup/deb-packages-dev.txt
@@ -2,20 +2,21 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# Deb Dev Package Requirements
# ============================================================================ #
-build-essential
+libldap2-dev
+libpq-dev # postgres pg_config
+# needed to build python-snappy for avro module
+libsnappy-dev
+
+# needed to build python-krbV and cloudera/thrift_sasl
libsasl2-dev
-python-dev
-unzip
-wget
-zip
diff --git a/setup/deb-packages-optional.txt b/setup/deb-packages-optional.txt
new file mode 100644
index 000000000..cd797f747
--- /dev/null
+++ b/setup/deb-packages-optional.txt
@@ -0,0 +1,16 @@
+#
+# Author: Hari Sekhon
+# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying LICENSE file
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# Deb Optional Package Requirements
+# ============================================================================ #
+
+yamllint
diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt
new file mode 100644
index 000000000..da4eea8c8
--- /dev/null
+++ b/setup/deb-packages-pip.txt
@@ -0,0 +1,33 @@
+#
+# Author: Hari Sekhon
+# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying LICENSE file
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# Deb PIP Package Requirements
+# ============================================================================ #
+
+python-avro
+python-dicttoxml
+python-git
+python-happybase
+python-humanize
+python-jinja2
+python-krbv
+python-ldap
+python-ldif3
+python-numpy
+python-psutil
+python-psycopg2
+python-sh
+python-snappy
+python-thrift
+python-xmltodict
+#python-bitarray
+#python-flask
diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt
old mode 100755
new mode 100644
index c754b90a7..4a028e4b1
--- a/setup/deb-packages.txt
+++ b/setup/deb-packages.txt
@@ -2,44 +2,20 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# Deb Package Requirements
# ============================================================================ #
-# needed to fetch the library submodule at end of build
-git
-
-python
-python-pip
-python-setuptools
-python-six
-unzip
-wget
-zip
-
-# needed for tests/test_spark* and ambari_cancel_all_requests.sh
-curl
-
-# either of these should do to ensure ping command is present for find_active_server.py's ping mode
-iputils-ping
-#inetutils-ping
-
# for htpasswd for docker registry authenticated checks
apache2-utils
-# needed for serf test's 'uptime' command
-procps
-
-# needed to build python-snappy for avro module
-libsnappy-dev
-
# installs 343MB of dependencies - install by hand if needed
#ffmpeg
@@ -51,3 +27,22 @@ libsnappy-dev
#ipython-notebook || :
#which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk
+
+# =====================================
+# installed by bash-tools submodule now
+
+# needed for tests/test_spark* and ambari_cancel_all_requests.sh
+#curl
+
+# either of these should do to ensure ping command is present for find_active_server.py's ping mode
+#iputils-ping
+##inetutils-ping
+
+# for anonymize_parallel.sh
+#parallel
+
+# needed for serf test's 'uptime' command
+#procps
+
+#zip
+#unzip
diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json
new file mode 100644
index 000000000..ce63bf63d
--- /dev/null
+++ b/setup/gocd_config_repo.json
@@ -0,0 +1,26 @@
+{
+ "id": "DevOps-Python-tools",
+ "plugin_id": "yaml.config.plugin",
+ "material": {
+ "type": "git",
+ "attributes": {
+ "url": "https://github.com/HariSekhon/DevOps-Python-tools",
+ "branch": "master",
+ "auto_update": true
+ }
+ },
+ "configuration": [
+ {
+ "key": "file_pattern",
+ "value": "cicd/*.gocd.y*ml"
+ }
+ ],
+ "rules": [
+ {
+ "directive": "allow",
+ "action": "*",
+ "type": "*",
+ "resource": "*"
+ }
+ ]
+}
diff --git a/setup/jenkins-job.xml b/setup/jenkins-job.xml
new file mode 100644
index 000000000..2c9da2b6b
--- /dev/null
+++ b/setup/jenkins-job.xml
@@ -0,0 +1,57 @@
+
+
+
+
+
+
+
+ hudson.triggers.SCMTrigger
+ hudson.triggers.TimerTrigger
+
+
+
+
+
+
+ false
+
+
+
+ https://github.com/HariSekhon/DevOps-Python-tools/
+
+
+
+
+
+ H 10 * * 1-5
+
+
+ H/2 * * * *
+ false
+
+
+
+
+
+
+ 2
+
+
+ https://github.com/HariSekhon/DevOps-Python-tools
+
+
+
+
+ */master
+
+
+ false
+
+
+
+ Jenkinsfile
+ true
+
+
+ false
+
\ No newline at end of file
diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt
old mode 100755
new mode 100644
index 0ccd31ac3..f36770476
--- a/setup/rpm-packages-dev.txt
+++ b/setup/rpm-packages-dev.txt
@@ -2,22 +2,24 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# RPM Dev Package Requirements
# ============================================================================ #
-cyrus-sasl-devel
-gcc
gcc-c++
-git
-python-devel
-unzip
-wget
-zip
+# needed to build python-krbV and cloudera/thrift_sasl
+cyrus-sasl-devel
+krb5-devel
+openldap-devel
+openssl-devel
+
+# moved to optional to account for changed package names on CentOS 8
+# needed to build python-snappy for avro module
+#snappy-devel
diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt
new file mode 100644
index 000000000..b1bb97d9e
--- /dev/null
+++ b/setup/rpm-packages-optional.txt
@@ -0,0 +1,27 @@
+#
+# Author: Hari Sekhon
+# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying LICENSE file
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# RPM Optional Package Requirements
+# ============================================================================ #
+
+yamllint
+
+# CentOS <= 7
+snappy-devel
+# CentOS 8
+csnappy-devel
+
+# postgres pg_config
+# CentOS 8
+libpq-devel
+# CentOS 7
+postgresql-devel
diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt
new file mode 100644
index 000000000..2a33f4bc5
--- /dev/null
+++ b/setup/rpm-packages-pip.txt
@@ -0,0 +1,32 @@
+#
+# Author: Hari Sekhon
+# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying LICENSE file
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# RPM PIP Package Requirements
+# ============================================================================ #
+
+GitPython
+python-humanize
+python-jinja2
+python-krbV
+python-ldap
+python-thrift
+python-xmltodict
+python2-numpy
+python2-psutil
+#python-flask
+#python-markupsafe
+#python2-bitarray
+
+python2-psycopg2
+python3-psycopg2
+
+python3-snappy
diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt
old mode 100755
new mode 100644
index ba28993d9..7f6263457
--- a/setup/rpm-packages.txt
+++ b/setup/rpm-packages.txt
@@ -2,35 +2,37 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
# RPM Package Requirements
# ============================================================================ #
-git
-python-pip
-python-setuptools
-
-# needed for tests/test_spark* and ambari_cancel_all_requests.sh
-curl
-
-# for ping mode of find_active_server.py
-iputils
+java
# for htpasswd for docker registry authenticated checks
-https-tools
+httpd-tools
# needed to build pyhs2
# libgsasl-devel saslwrapper-devel
#cyrus-sasl-devel
-# needed to build python-snappy for avro module
-snappy-devel
+# =====================================
+# installed by bash-tools submodule now
+
+# needed for tests/test_spark* and ambari_cancel_all_requests.sh
+#curl
+
+# for ping mode of find_active_server.py
+#iputils
+
+# for anonymize_parallel.sh
+#parallel
-#which java || $(SUDO) yum install -y java
+#unzip
+#zip
diff --git a/sonar-project.properties b/sonar-project.properties
index 75e1bf83c..319cc6897 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -1,32 +1,42 @@
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
-# Date: 2016-07-19 18:18:27 +0100 (Tue, 19 Jul 2016)
+# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
-sonar.projectName=DevOps Python Tools
-sonar.projectKey=pytools
+# ============================================================================ #
+# S o n a r Q u b e
+# ============================================================================ #
+
+sonar.host.url=https://sonarcloud.io
+
+# Required metadata
+sonar.organization=harisekhon
+sonar.projectName=DevOps-Python-tools
+sonar.projectKey=HariSekhon_DevOps-Python-tools
sonar.projectVersion=1.0
-sonar.projectDescription=Python / Jython Tools
+sonar.projectDescription=DevOps-Python-tools
-sonar.links.homepage=https://github.com/harisekhon/devops-python-tools
-sonar.links.scm=https://github.com/harisekhon/devops-python-tools
-sonar.links.issue=https://github.com/harisekhon/devops-python-tools/issues
-sonar.links.ci=https://travis-ci.org/HariSekhon/devops-python-tools
+sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools
+sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools
+sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues
+sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions
+# directories to scan (defaults to sonar-project.properties dir otherwise)
sonar.sources=.
#sonar.language=py
sonar.sourceEncoding=UTF-8
-sonar.exclusions=**/tests/spark*/**/*
+#sonar.exclusions=**/tests/**
+sonar.exclusions=**/zookeeper-*/**/*
diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py
index 10fd55e88..8528af1a6 100755
--- a/spark_avro_to_parquet.py
+++ b/spark_avro_to_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,6 +52,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py
index 955b395e7..d41717b4f 100755
--- a/spark_csv_to_avro.py
+++ b/spark_csv_to_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -57,14 +57,15 @@
'com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
-from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import
+from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import
from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error
__author__ = 'Hari Sekhon'
-__version__ = '0.8.0'
+__version__ = '0.8.1'
class SparkCSVToAvro(CLI):
@@ -166,7 +167,6 @@ def create_struct(arg):
die("Spark version couldn't be determined. " + support_msg('pytools'))
# pylint: disable=invalid-name
- df = None
if isMinVersion(spark_version, 1.4):
if has_header and not schema:
log.info('inferring schema from CSV headers')
@@ -198,5 +198,6 @@ def create_struct(arg):
# the databricks avro driver
df.write.format('com.databricks.spark.avro').save(avro_dir)
+
if __name__ == '__main__':
SparkCSVToAvro().main()
diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py
index f3f6909e6..e70f014f6 100755
--- a/spark_csv_to_parquet.py
+++ b/spark_csv_to_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -49,14 +49,15 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-csv_2.11:1.5.0 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
-from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import
+from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import
from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error
__author__ = 'Hari Sekhon'
-__version__ = '0.8.0'
+__version__ = '0.8.1'
class SparkCSVToParquet(CLI):
@@ -157,9 +158,7 @@ def create_struct(arg):
if not isVersionLax(spark_version):
die("Spark version couldn't be determined. " + support_msg('pytools'))
- # pylint: disable=invalid-name
-
- df = None
+ # pylint: disable=invalid-name
if isMinVersion(spark_version, 1.4):
if has_header and not schema:
log.info('inferring schema from CSV headers')
diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py
index 40c52ac96..db9d069ed 100755
--- a/spark_json_to_avro.py
+++ b/spark_json_to_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,6 +52,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
@@ -109,7 +110,6 @@ def run(self):
die("Spark version couldn't be determined. " + support_msg('pytools'))
# pylint: disable=invalid-name
- df = None
if isMinVersion(spark_version, 1.4):
df = sqlContext.read.json(json_file)
else:
diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py
index 41ab27e0d..6976ce153 100755
--- a/spark_json_to_parquet.py
+++ b/spark_json_to_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -43,6 +43,7 @@
print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
sys.exit(4)
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py
index d6e49f060..030816963 100755
--- a/spark_parquet_to_avro.py
+++ b/spark_parquet_to_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,6 +52,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
diff --git a/sql b/sql
new file mode 160000
index 000000000..30d80bbd3
--- /dev/null
+++ b/sql
@@ -0,0 +1 @@
+Subproject commit 30d80bbd33f0e6c26c230f26575cb671c41c3e7d
diff --git a/strip_ansi_escape_codes.py b/strip_ansi_escape_codes.py
index b828faaa8..24738194a 100755
--- a/strip_ansi_escape_codes.py
+++ b/strip_ansi_escape_codes.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2018-09-09 23:06:06 +0100 (Sun, 09 Sep 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish # pylint: disable=line-too-long
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -42,7 +42,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.1'
+__version__ = '0.2'
# pylint: disable=too-few-public-methods
@@ -72,11 +72,11 @@ def run(self):
die("path '%s' could not be determined as either a file or directory" % arg)
for filename in self.args:
if filename == '-':
- for line in sys.stdin.readlines():
+ for line in sys.stdin:
print(strip_ansi_escape_codes(line), end='')
else:
- with open(filename) as _:
- for line in _.readlines():
+ with open(filename) as filehandle:
+ for line in filehandle:
print(strip_ansi_escape_codes(line), end='')
diff --git a/teamcity/.teamcity.vcs.json b/teamcity/.teamcity.vcs.json
new file mode 100644
index 000000000..be61f126b
--- /dev/null
+++ b/teamcity/.teamcity.vcs.json
@@ -0,0 +1,57 @@
+{
+ "id": "TeamCity",
+ "name": "TeamCity",
+ "vcsName": "jetbrains.git",
+ "href": "/httpAuth/app/rest/vcs-roots/id:TeamCity",
+ "project": {
+ "id": "_Root",
+ "name": "",
+ "description": "Contains all other projects",
+ "href": "/httpAuth/app/rest/projects/id:_Root",
+ "webUrl": "http://localhost:8111/project.html?projectId=_Root"
+ },
+ "properties": {
+ "count": 9,
+ "property": [
+ {
+ "name": "agentCleanFilesPolicy",
+ "value": "ALL_UNTRACKED"
+ },
+ {
+ "name": "agentCleanPolicy",
+ "value": "ON_BRANCH_CHANGE"
+ },
+ {
+ "name": "authMethod",
+ "value": "ANONYMOUS"
+ },
+ {
+ "name": "branch",
+ "value": "master"
+ },
+ {
+ "name": "ignoreKnownHosts",
+ "value": "true"
+ },
+ {
+ "name": "submoduleCheckout",
+ "value": "CHECKOUT"
+ },
+ {
+ "name": "url",
+ "value": "https://github.com/HariSekhon/TeamCity-CI"
+ },
+ {
+ "name": "useAlternates",
+ "value": "true"
+ },
+ {
+ "name": "usernameStyle",
+ "value": "USERID"
+ }
+ ]
+ },
+ "vcsRootInstances": {
+ "href": "/httpAuth/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)"
+ }
+}
diff --git a/teamcity/.teamcity.vcs.oauth.json b/teamcity/.teamcity.vcs.oauth.json
new file mode 100644
index 000000000..f6f41ed7b
--- /dev/null
+++ b/teamcity/.teamcity.vcs.oauth.json
@@ -0,0 +1,61 @@
+{
+ "id": "TeamCity",
+ "name": "TeamCity",
+ "vcsName": "jetbrains.git",
+ "href": "/httpAuth/app/rest/vcs-roots/id:TeamCity",
+ "project": {
+ "id": "_Root",
+ "name": "",
+ "description": "Contains all other projects",
+ "href": "/httpAuth/app/rest/projects/id:_Root",
+ "webUrl": "http://localhost:8111/project.html?projectId=_Root"
+ },
+ "properties": {
+ "count": 9,
+ "property": [
+ {
+ "name": "agentCleanFilesPolicy",
+ "value": "ALL_UNTRACKED"
+ },
+ {
+ "name": "agentCleanPolicy",
+ "value": "ON_BRANCH_CHANGE"
+ },
+ {
+ "name": "authMethod",
+ "value": "PASSWORD"
+ },
+ {
+ "name": "branch",
+ "value": "master"
+ },
+ {
+ "name": "ignoreKnownHosts",
+ "value": "true"
+ },
+ {
+ "name": "submoduleCheckout",
+ "value": "CHECKOUT"
+ },
+ {
+ "name": "url",
+ "value": "https://github.com/HariSekhon/TeamCity-CI"
+ },
+ {
+ "name": "useAlternates",
+ "value": "true"
+ },
+ {
+ "name": "username",
+ "value": "HariSekhon"
+ },
+ {
+ "name": "usernameStyle",
+ "value": "USERID"
+ }
+ ]
+ },
+ "vcsRootInstances": {
+ "href": "/httpAuth/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)"
+ }
+}
diff --git a/teamcity/.teamcity.vcs.ssh.json b/teamcity/.teamcity.vcs.ssh.json
new file mode 100644
index 000000000..1b780675a
--- /dev/null
+++ b/teamcity/.teamcity.vcs.ssh.json
@@ -0,0 +1,65 @@
+{
+ "id": "TeamCity",
+ "name": "TeamCity",
+ "vcsName": "jetbrains.git",
+ "href": "/app/rest/vcs-roots/id:TeamCity",
+ "project": {
+ "id": "_Root",
+ "name": "",
+ "description": "Contains all other projects",
+ "href": "/app/rest/projects/id:_Root",
+ "webUrl": "http://localhost:8111/project.html?projectId=_Root"
+ },
+ "properties": {
+ "count": 11,
+ "property": [
+ {
+ "name": "agentCleanFilesPolicy",
+ "value": "ALL_UNTRACKED"
+ },
+ {
+ "name": "agentCleanPolicy",
+ "value": "ON_BRANCH_CHANGE"
+ },
+ {
+ "name": "authMethod",
+ "value": "TEAMCITY_SSH_KEY"
+ },
+ {
+ "name": "branch",
+ "value": "refs/heads/master"
+ },
+ {
+ "name": "ignoreKnownHosts",
+ "value": "true"
+ },
+ {
+ "name": "submoduleCheckout",
+ "value": "CHECKOUT"
+ },
+ {
+ "name": "teamcitySshKey",
+ "value": "VCS SSH Key"
+ },
+ {
+ "name": "url",
+ "value": "github.com:HariSekhon/TeamCity-CI"
+ },
+ {
+ "name": "useAlternates",
+ "value": "true"
+ },
+ {
+ "name": "username",
+ "value": "git"
+ },
+ {
+ "name": "usernameStyle",
+ "value": "USERID"
+ }
+ ]
+ },
+ "vcsRootInstances": {
+ "href": "/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)"
+ }
+}
diff --git a/templates b/templates
new file mode 160000
index 000000000..f59532dd6
--- /dev/null
+++ b/templates
@@ -0,0 +1 @@
+Subproject commit f59532dd67a583be3522814a15024a561b12c5ec
diff --git a/tests/all.sh b/tests/all.sh
index 140d28926..dbac7b8fb 100755
--- a/tests/all.sh
+++ b/tests/all.sh
@@ -4,23 +4,24 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+# shellcheck disable=SC1090
. "$srcdir/utils.sh"
# imported by utils.sh above
-#. "$srcdir/../bash-tools/utils.sh"
+#. "$srcdir/../bash-tools/lib/utils.sh"
export PROJECT=pytools
@@ -28,11 +29,16 @@ section "Running PyTools ALL"
# runs against . by default
cd "$srcdir/..";
-bash-tools/all.sh
+# shellcheck disable=SC1090
+# has to be included so that isExcluded function is inherited
+. "$srcdir/../bash-tools/checks/check_all.sh"
-exit 0
-
-bash-tools/run_tests.sh
+#tests/test_yamllint.sh
# do help afterwards for Spark to be downloaded, and then help will find and use downloaded spark for SPARK_HOME
+exit 0
+
+# pyspark not found
tests/help.sh
+
+bash-tools/checks/run_tests.sh
diff --git a/tests/check.sh b/tests/check.sh
index 4541f684b..b3dba3c12 100755
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:47:43 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -21,12 +21,12 @@ check(){
msg=$2
echo
hr2
- echo $msg
+ echo "$msg"
hr2
echo
- echo cmd: $cmd
+ echo "cmd: $cmd"
echo
- if eval $cmd; then
+ if eval "$cmd"; then
echo
echo "SUCCESS"
else
diff --git a/tests/compile.sh b/tests/compile.sh
index e235cfd70..2890b07a3 100755
--- a/tests/compile.sh
+++ b/tests/compile.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,17 +19,18 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
-. ./tests/utils.sh
+# shellcheck disable=SC1090
+. "$srcdir/utils.sh"
hr
echo "Compiling all Python files"
hr
echo
-for x in $(find . -iname '*.py' -o -iname '*.jy'); do
- isExcluded "$x" && continue
- echo "compiling $x"
- python -m py_compile $x
-done
+while read -r filename; do
+ isExcluded "$filename" && continue
+ echo "compiling $filename"
+ python -m py_compile "$filename"
+done < <(find . -iname '*.py' -o -iname '*.jy')
echo
echo
diff --git a/tests/data/add_ou.ldif b/tests/data/add_ou.ldif
index c175b4bba..c9bc193b2 100644
--- a/tests/data/add_ou.ldif
+++ b/tests/data/add_ou.ldif
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-08-03 17:44:12 +0200 (Thu, 03 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
dn: OU=Hadoop-Cluster-1,OU=Hadoop,DC=harisekhon,DC=co,DC=uk
diff --git a/tests/data/ldap.ldif b/tests/data/ldap.ldif
new file mode 100644
index 000000000..b9786d768
--- /dev/null
+++ b/tests/data/ldap.ldif
@@ -0,0 +1,70 @@
+# extended LDIF
+#
+# LDAPv3
+# base with scope subtree
+# filter: (objectclass=*)
+# requesting: ALL
+#
+
+# example.org
+dn: dc=example,dc=org
+objectClass: top
+objectClass: dcObject
+objectClass: organization
+o: Example Inc.
+dc: example
+
+# admin, example.org
+dn: cn=admin,dc=example,dc=org
+objectClass: simpleSecurityObject
+objectClass: organizationalRole
+cn: admin
+description: LDAP administrator
+userPassword:: e1NTSEF9TnpTTnRqbzZUUlgrM2c3Y3c4VHU3eVpCOXhaaDJtN00=
+
+# Users, example.org
+dn: ou=Users,dc=example,dc=org
+objectClass: organizationalUnit
+objectClass: top
+ou: Users
+
+# Groups, example.org
+dn: ou=Groups,dc=example,dc=org
+objectClass: organizationalUnit
+objectClass: top
+ou: Groups
+
+# Dept1, Groups, example.org
+dn: ou=Dept1,ou=Groups,dc=example,dc=org
+objectClass: organizationalUnit
+objectClass: top
+ou: Dept1
+
+# DeptA, Dept1, Groups, example.org
+dn: ou=DeptA,ou=Dept1,ou=Groups,dc=example,dc=org
+objectClass: organizationalUnit
+objectClass: top
+ou: DeptA
+
+# hari, Users, example.org
+dn: cn=hari,ou=Users,dc=example,dc=org
+cn: hari
+objectClass: organizationalPerson
+objectClass: person
+objectClass: top
+sn: sekhon
+memberOf: cn=SuperAdmins,ou=DeptA,ou=Dept1,ou=Groups,dc=example,dc=org
+
+# SuperAdmins, DeptA, Dept1, Groups, example.org
+dn: cn=SuperAdmins,ou=DeptA,ou=Dept1,ou=Groups,dc=example,dc=org
+cn: SuperAdmins
+member: cn=hari,ou=Users,dc=example,dc=org
+objectClass: groupOfNames
+objectClass: top
+
+# search result
+#search: 2
+#result: 0 Success
+
+# numResponses: 9
+# numEntries: 8
diff --git a/tests/data/ldap_download.sh b/tests/data/ldap_download.sh
new file mode 100755
index 000000000..8c7f4a158
--- /dev/null
+++ b/tests/data/ldap_download.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+ldapsearch -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -b dc=example,dc=org > "$srcdir/ldap.ldif"
diff --git a/tests/data/ldap_upload.sh b/tests/data/ldap_upload.sh
new file mode 100755
index 000000000..60cdeb5ab
--- /dev/null
+++ b/tests/data/ldap_upload.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# -a add
+# -c continue on error (if already exists)
+
+# Insufficient Permissions :-/
+#ldapmodify -a -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap_memberof_overlay.ldif"
+
+# add
+#ldapmodify -a -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap.ldif" || :
+ldapadd -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap.ldif"
+
+# modify
+#ldapmodify -c -x -H ldap://localhost -D cn=admin,dc=example,dc=org -w admin -f "$srcdir/ldap.ldif"
diff --git a/tests/data/test.toml b/tests/data/test.toml
new file mode 100644
index 000000000..64d942504
--- /dev/null
+++ b/tests/data/test.toml
@@ -0,0 +1,25 @@
+
+title = "TOML Example"
+
+[owner]
+name = "Hari Sekhon"
+
+[database]
+server = "192.168.1.1"
+ports = [ 8001, 8001, 8002,]
+connection_max = 5000
+enabled = true
+
+[clients]
+data = [ [ "gamma", "delta",], [ 1, 2,],]
+hosts = [ "alpha", "omega",]
+
+[servers.alpha]
+ip = "10.0.0.1"
+dc = "eqdc10"
+
+[servers.beta]
+
+ip = "10.0.0.2"
+dc = "eqdc10"
+
diff --git a/tests/data/uk_marriage_rates_2022.xslx b/tests/data/uk_marriage_rates_2022.xslx
new file mode 100644
index 000000000..54efc3321
Binary files /dev/null and b/tests/data/uk_marriage_rates_2022.xslx differ
diff --git a/tests/docker/apache-drill-docker-compose.yml b/tests/docker/apache-drill-docker-compose.yml
index 50c1de358..d5ede5b8b 100644
--- a/tests/docker/apache-drill-docker-compose.yml
+++ b/tests/docker/apache-drill-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 16:18:36 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/common.yml b/tests/docker/common.yml
index 4c4236d1c..0b2a2d133 100644
--- a/tests/docker/common.yml
+++ b/tests/docker/common.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 15:16:43 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/elasticsearch-docker-compose.yml b/tests/docker/elasticsearch-docker-compose.yml
index bd6d9c190..4838ee391 100644
--- a/tests/docker/elasticsearch-docker-compose.yml
+++ b/tests/docker/elasticsearch-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/elasticsearch-elastic.co-docker-compose.yml b/tests/docker/elasticsearch-elastic.co-docker-compose.yml
index 992448ba6..6b7095ab6 100644
--- a/tests/docker/elasticsearch-elastic.co-docker-compose.yml
+++ b/tests/docker/elasticsearch-elastic.co-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/hadoop-docker-compose.yml b/tests/docker/hadoop-docker-compose.yml
index 63e9f53de..157d45c63 100644
--- a/tests/docker/hadoop-docker-compose.yml
+++ b/tests/docker/hadoop-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 21:25:07 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
@@ -18,7 +18,7 @@ services:
extends:
file: common.yml
service: common
- image: harisekhon/hadoop:${VERSION:-latest}
+ image: harisekhon/hadoop-dev:${VERSION:-latest}
ports:
#- 8020
- 8042
@@ -31,6 +31,8 @@ services:
- 50070
- 50075
- 50090
+ volumes:
+ - ../..:/py
networks:
- hadoop
diff --git a/tests/docker/hbase-docker-compose.yml b/tests/docker/hbase-docker-compose.yml
index 04029f2bb..8ab70732b 100644
--- a/tests/docker/hbase-docker-compose.yml
+++ b/tests/docker/hbase-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 22:13:19 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/presto-dev-docker-compose.yml b/tests/docker/presto-dev-docker-compose.yml
index 835b126f1..aaf4872f4 100644
--- a/tests/docker/presto-dev-docker-compose.yml
+++ b/tests/docker/presto-dev-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.1'
diff --git a/tests/docker/presto-docker-compose.yml b/tests/docker/presto-docker-compose.yml
index bb09ee3ed..2b9cc99ed 100644
--- a/tests/docker/presto-docker-compose.yml
+++ b/tests/docker/presto-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.1'
diff --git a/tests/docker/registry-docker-compose.yml b/tests/docker/registry-docker-compose.yml
index 7a66c2e3f..5fae25ab1 100644
--- a/tests/docker/registry-docker-compose.yml
+++ b/tests/docker/registry-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-09-12 17:27:50 +0200 (Tue, 12 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/solrcloud-docker-compose.yml b/tests/docker/solrcloud-docker-compose.yml
index 7799417a7..9cf4885fa 100644
--- a/tests/docker/solrcloud-docker-compose.yml
+++ b/tests/docker/solrcloud-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-13 14:01:04 +0000 (Tue, 13 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/excluded.sh b/tests/excluded.sh
index 7bbe7c34e..7535d3be6 100755
--- a/tests/excluded.sh
+++ b/tests/excluded.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# intended only to be sourced by utils.sh
@@ -22,16 +22,19 @@ set -eu
isExcluded(){
local prog="$1"
+ # shellcheck disable=SC2049
[[ "$prog" =~ ^\* ]] && return 0
+ [[ "$prog" =~ spark_.*\.py ]] && return 0
+ [[ "$prog" =~ \.jy ]] && return 0
+ [[ "$prog" =~ hdfs_find_replication_factor_1\.py ]] && return 0 # python-krbV doesn't build on Python 3
#[[ $prog =~ ipython-notebook ]] && return 0
- #[[ $prog =~ ambari_blueprints.py ]] && return 0
# this external git check is expensive, skip it when in CI as using fresh git checkouts
is_CI && return 1
# we no longer include spark inside pytools docker image due to bloat, so skip tests
if is_inside_docker && [[ "$prog" =~ spark_* ]]; then
return 0
fi
- if which git &>/dev/null; then
+ if type -P git &>/dev/null; then
commit="$(git log "$prog" | head -n1 | grep 'commit')"
if [ -z "$commit" ]; then
return 0
diff --git a/tests/help.sh b/tests/help.sh
index efac9efef..09b48dfe0 100755
--- a/tests/help.sh
+++ b/tests/help.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,18 +19,27 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
-for x in $(echo *.py 2>/dev/null); do
+# shellcheck disable=SC2068
+for x in ${@:-$(echo ./*.py 2>/dev/null)}; do
isExcluded "$x" && continue
set +e
- echo ./$x --help
- ./$x --help # >/dev/null
+ echo "$x --help"
+ "$x" --help # >/dev/null
status=$?
set -e
echo; hr
if [ $status = 0 ]; then
- [[ $x =~ ambari_blueprints ]] && continue
+ [[ "$x" =~ ambari_blueprints.py$ ]] && continue
+ [[ "$x" =~ pythonpath.py$ ]] && continue
+ [[ "$x" =~ aws_s3_presign.py$ ]] && continue
+ elif [ $status = 1 ]; then
+ if [[ "$x" =~ hdfs_find_replication_factor_1.py$ ]] &&
+ ! python -c 'import krbV'; then # best effort, not available on Mac any more
+ continue
+ fi
fi
[ $status = 3 ] || { echo "status code for $x --help was $status not expected 3"; exit 1; }
done
diff --git a/tests/python3.sh b/tests/python3.sh
index eeaf5f9f3..b231f8df4 100755
--- a/tests/python3.sh
+++ b/tests/python3.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,6 +19,7 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
pip install caniusepython3
diff --git a/tests/syntax.sh b/tests/syntax.sh
index 8008e1fbd..5299ce323 100755
--- a/tests/syntax.sh
+++ b/tests/syntax.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,20 +19,21 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
-for x in $(echo *.py *.jy 2>/dev/null); do
- isExcluded "$x" && continue
- if which flake8 &> /dev/null; then
- echo "flake8 $x"
- flake8 --max-line-length=120 --statistics $x
+while read -r prog; do
+ isExcluded "$prog" && continue
+ if type -P flake8 &> /dev/null; then
+ echo "flake8 $prog"
+ flake8 --max-line-length=120 --statistics "$prog"
echo; hr; echo
fi
for y in pyflakes pychecker; do
- if which $y &>/dev/null; then
- echo "$y $x"
- $y $x
+ if type -P "$y" &>/dev/null; then
+ echo "$y $prog"
+ "$y" "$prog"
echo; hr; echo
fi
done
-done
+done < <(find . -type f -name '*.py' -o -type f -name '*.jy')
diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py
new file mode 100755
index 000000000..4bd3aebc0
--- /dev/null
+++ b/tests/test_anonymize.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import re
+#import StringIO
+import subprocess
+from subprocess import PIPE
+import sys
+
+srcdir = os.path.abspath(os.path.dirname(__file__))
+
+anonymize_test_sh = os.path.join(srcdir, 'test_anonymize.sh')
+
+anonymize = '{}/../anonymize.py'.format(srcdir)
+
+src = {}
+dest = {}
+
+src_regex = re.compile(r'^\s*src\[(\d+)\]=["\'](.+)["\']\s*$')
+dest_regex = re.compile(r'^\s*dest\[(\d+)\]=["\'](.+)["\']\s*$')
+args_regex = re.compile(r'^\s*args=["\'](.+)["\']\s*$')
+
+def normalize_text(text):
+ text = text.replace(r'\"', '"')
+ text = text.replace(r"\'", "'")
+ return text
+
+def run():
+ #test_input = StringIO.StringIO()
+ #test_input.write('\n'.join(src))
+ global src # pylint: disable=global-statement
+ global dest # pylint: disable=global-statement
+ src = {int(k) : v for k, v in src.items()}
+ dest = {int(k) : v for k, v in dest.items()}
+ src_keys = sorted(src)
+ test_input = '\n'.join([src[_] for _ in src_keys])
+
+ print('running anonymize tests using: {} {}'.format(anonymize, args))
+ cmd = [anonymize] + args.split()
+ process = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE)
+ # encode as bytes for Python 3 :-/
+ test_input = str.encode(test_input, 'utf-8')
+ (stdout, _) = process.communicate(input=test_input)
+ index = 0
+ # convert bytes to string
+ stdout = stdout.decode("utf-8")
+ # pylint: disable=redefined-outer-name
+ for line in stdout.split('\n'):
+ key = src_keys[index]
+ _input = src[key]
+ expected = dest[key]
+ if line != expected:
+ print('FAILED to anonymize line during test {}'.format(key))
+ print('input: {}'.format(_input))
+ print('expected: {}'.format(expected))
+ print('got: {}'.format(line))
+ sys.exit(1)
+ print('SUCCEEDED anonymization test {}'.format(key))
+ index += 1
+
+with open(anonymize_test_sh) as filehandle:
+ for line in filehandle:
+ src_match = src_regex.match(line)
+ if src_match:
+ key = src_match.group(1)
+ if key in src:
+ raise AssertionError('Duplicate key index src[{}]'.format(key))
+ value = src_match.group(2)
+ value = normalize_text(value)
+ src[key] = value
+ dest_match = dest_regex.match(line)
+ if dest_match:
+ key = dest_match.group(1)
+ if key in dest:
+ raise AssertionError('Duplicate key index dest[{}]'.format(key))
+ value = dest_match.group(2)
+ value = normalize_text(value)
+ dest[key] = value
+ args_match = args_regex.match(line)
+ if args_match:
+ args = args_match.group(1)
+ run()
+ src = {}
+ dest = {}
diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh
index 116cfad8f..9aebe794f 100755
--- a/tests/test_anonymize.sh
+++ b/tests/test_anonymize.sh
@@ -4,29 +4,30 @@
# Author: Hari Sekhon
# Date: 2015-07-28 18:47:41 +0100 (Tue, 28 Jul 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-test_nums="${@:-}"
-parallel=""
-if [ "$test_nums" = "p" ]; then
- parallel="1"
- test_nums=""
-fi
+test_nums="${*:-}"
+#parallel=""
+#if [ "$test_nums" = "p" ]; then
+# parallel="1"
+# test_nums=""
+#fi
cd "$srcdir/..";
+# shellcheck disable=SC1091
. ./tests/utils.sh
section "Anonymize"
@@ -39,35 +40,6 @@ start_time="$(start_timer "$anonymize")"
# Custom Tests
# ============================================================================ #
-if [ -z "$test_nums" ]; then
- echo
- echo "Running Custom Tests:"
- echo
- echo "checking file args:"
- run++
- if [ `$anonymize -ae README.md | wc -l` -gt 100 ]; then
- echo "SUCCEEDED - anonymized README.md > 100 lines"
- else
- echo "FAILED - suspicious README.md file arg result came to <= 100 lines"
- exit 1
- fi
- hr
-
- echo "testing --email replacememnt format:"
- run_grep "@" $anonymize --email <<< "hari@domain.com"
- run_grep "@" $anonymize -E <<< "hari@domain.com"
-
- echo "testing --ip-prefix behaviour:"
- run_grep ".1" $anonymize --ip-prefix <<< "4.3.2.1"
-
- echo "testing --hash-hostnames:"
- run_grep "^http://[a-f0-9]{12}:80/path$" $anonymize --hash-hostnames <<< "http://test.domain.com:80/path"
- run_grep '^\\\\[a-f0-9]{12}\\mydir$' $anonymize --hash-hostnames <<< '\\test.domain.com\mydir'
- run_grep '-host [a-f0-9]{12}' $anonymize --hash-hostnames <<< '-host blah'
-fi
-
-# ============================================================================ #
-
src[0]="2015-11-19 09:59:59,893 - Execution of 'mysql -u root --password=somep@ssword! -h myHost.internal -s -e \"select version();\"' returned 1. ERROR 2003 (HY000): Can't connect to MySQL server on 'host.domain.com' (111)"
dest[0]="2015-11-19 09:59:59,893 - Execution of 'mysql -u root --password= -h -s -e \"select version();\"' returned 1. ERROR 2003 (HY000): Can't connect to MySQL server on '' (111)"
@@ -340,8 +312,8 @@ dest[84]="ldap:///dc=,dc=??sub?(givenName=)"
src[85]="ip-1-2-3-4-5"
dest[85]="ip-1-2-3-4-5"
-src[86]="dip-1-2-3-4-5"
-dest[86]="dip-1-2-3-4-5"
+src[86]="dip-1-2-3-4"
+dest[86]="dip-1-2-3-4"
src[87]="1.2.3.4.5"
dest[87]="1.2.3.4.5"
@@ -350,7 +322,7 @@ src[88]="-host blah"
dest[88]="-host "
src[89]="--hostname=blah --anotherswitch"
-dest[89]="-hostname= --anotherswitch"
+dest[89]="--hostname= --anotherswitch"
src[90]="host=test"
dest[90]="host="
@@ -365,22 +337,186 @@ dest[92]="hostname=;port=92"
src[93]="hari@domain.com, hari@domain2.co.uk"
dest[93]="@, @"
+src[94]="spark://hari@192.168.0.1"
+dest[94]="spark://@"
+
+src[95]="/usr/hdp/2.6.2.0-123/blah"
+dest[95]="/usr/hdp/2.6.2.0-123/blah"
+
+src[96]="log4j-1.2.3.4.jar"
+dest[96]="log4j-1.2.3.4.jar"
+
+src[97]='MYDOMAIN\n2018-10-03 01:23:45'
+dest[97]='MYDOMAIN\n2018-10-03 01:23:45'
+
+src[98]='domain789012345\blah'
+dest[98]='\'
+
+src[99]='domain7890123456\blah'
+dest[99]='domain7890123456\blah'
+
+src[100]="-Dhost=blah"
+dest[100]="-Dhost="
+
+src[101]="-Ddomain.com=blah"
+dest[101]="-Ddomain.com=blah"
+
+src[102]="-Dhost.domain.com=blah"
+dest[102]="-Dhost.domain.com=blah"
# check escape codes get stripped if present (eg. if piping from grep --color-yes)
-#src[88]="some^[[01;31m^[[Khost^[[m^[[Kname:443"
-#src[88]="some\e[01;31m\e[Khost\e[m\e[K:443"
-src[99]="$(echo somehost:443 | grep --color=yes host)"
-dest[99]=":443"
+# breaks test_anonymize.py which doesn't eval this, so put it explicitly
+#src[103]="$(echo somehost:443 | grep --color=yes host)"
+src[103]="some[01;31m[Khost[m[K:443"
+dest[103]=":443"
+
+src[104]='..., "user": "blah", "group": "blah2", "host": "blah3", ...'
+dest[104]='..., "user": "", "group": "", "host": "", ...'
+
+src[105]='...,"owner":"blah","hostname":"blah2",...'
+dest[105]='...,"owner":"","hostname":"",...'
+
+src[106]="ambari-sudo.sh"
+dest[106]="ambari-sudo.sh"
+
+src[107]="hdfs://user/blah"
+dest[107]="hdfs:///"
+
+src[108]="hdfs:///user/blah"
+dest[108]="hdfs:///user/"
+
+src[109]="es.xpack.user=hari"
+dest[109]="es.xpack.user="
+
+src[110]="es.xpack.password=myp@ss!"
+dest[110]="es.xpack.password="
+
+src[111]="127.0.0.1"
+dest[111]="127.0.0.1"
+
+src[112]="travis token: Abc123"
+dest[112]="travis token: "
+
+src[113]="arn:aws:iam::123456789012:user/hari"
+dest[113]="arn:aws:iam:::user/"
+
+src[114]="arn:aws:iam::123456789012:group/hari"
+dest[114]="arn:aws:iam:::group/"
+
+src[115]="arn:aws:iam::123456789012:user/Development/product_1234/*"
+dest[115]="arn:aws:iam:::user//*"
+
+src[116]="arn:aws:iam::123456789012:group/Development/product_1234/*"
+dest[116]="arn:aws:iam:::group//*"
+
+src[117]="arn:aws:iam::123456789012:group/Development/product_1234/*"
+dest[117]="arn:aws:iam:::group//*"
+
+src[118]="arn:aws:s3:::my_corporate_bucket/Development/*"
+dest[118]="arn:aws:s3:::*"
+
+src[119]="AKIAIOSFODNN7EXAMPLE"
+dest[119]=""
+
+src[120]="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+dest[120]=""
+
+src[121]="AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/LTo6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3zrkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtpZ3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE"
+dest[121]=""
+
+src[122]="ASIAIOSFODNN7EXAMPLE"
+dest[122]=""
+
+src[123]="AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA=="
+dest[123]=""
+
+# security groups
+src[124]="sg-5f63c627"
+dest[124]=""
+
+src[125]="s3://myBucket/file.txt"
+dest[125]="s3://