diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..740e2f11c --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,89 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 16:19:35 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# A p p V e y o r C I +# ============================================================================ # + +# https://www.appveyor.com/docs/appveyor-yml/ + +image: Ubuntu + +# workaround for default JDK9 have old CA certs: +# +# https://github.com/appveyor/ci/issues/3833 +# +# https://www.appveyor.com/docs/getting-started-with-appveyor-for-linux/#configuring-language-stack +# +stack: jdk 15 + +skip_commits: + files: + - docs/* + - '**/*.md' + +# https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ +environment: + APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension + +# enable SSH session accessible via my public key +#init: +# - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e - + +# more useful at end to leverage .appveyor.yml tweaks like disabling broken mssql repo/dependencies, checking out project and building the core stuff happen first so we don't have to do all that manually in SSH session +on_finish: + # set this in Settings -> Environment dynamically instead of here + #- sh: export APPVEYOR_SSH_BLOCK=true + # + # workaround for https://github.com/appveyor/ci/issues/3373 + # and https://github.com/appveyor/ci/issues/3384 + # + # has since been added to AppVeyor's own scripts: + # + # https://github.com/appveyor/ci/pull/3385 + # + #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/install/install_openssh.sh' | bash -e - + # + # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/ + - sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi + +install: + # workaround for: + # Some packages could not be installed. This may mean that you have + # requested an impossible situation or if you are using the unstable + # distribution that some required packages have not yet been created + # or been moved out of Incoming. + # The following information may help to resolve the situation: + # + # The following packages have unmet dependencies: + # mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed + # E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages. + # DevOps-Python-tools/Makefile.in:272: recipe for target 'apt-packages' failed + # make[2]: *** [apt-packages] Error 123 + # make[2]: Leaving directory '/home/appveyor/projects/pylib' + # DevOps-Python-tools/Makefile.in:212: recipe for target 'system-packages' failed + # + # adding "|| :" to the end of these commands causes them to be silently ignored! + - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list + - sudo apt purge -yq --allow-change-held-packages mssql-server + # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411 + #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages + - setup/ci_bootstrap.sh + - make + +test_script: + - make test + +build: off diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 000000000..eb0d641fd --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-13 21:10:39 +0000 (Fri, 13 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# B u i l d K i t e C I +# ============================================================================ # + +# BuildKite Pipeline +# +# add this command to the UI and it will read the rest of the steps from here: +# +# - command: buildkite-agent pipeline upload + +# Yaml Anchors workaround to BuildKite's lack of global retries configuration - credit to Jason @ BuildKite for this workaround: + # +# https://forum.buildkite.community/t/reschedule-builds-on-other-agents-rather-than-fail-builds-when-agents-time-out-or-are-killed-machine-shut-down-or-put-to-sleep/1388/5 +# +anchors: + std_retries: &std_retries + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: 255 # Forced agent shutdown + limit: 2 + +steps: + - command: setup/ci_bootstrap.sh + label: ci bootstrap + timeout: 30 # brew can take 10 mins just to do a brew update + branches: master + <<: [*std_retries] + - wait + - command: make init + label: init + timeout: 2 + branches: master + <<: [*std_retries] + - wait + - command: make ci + label: build + timeout: 60 + branches: master + <<: [*std_retries] + - wait + - command: make test + label: test + timeout: 120 + branches: master + <<: [*std_retries] diff --git a/.checkov.yaml b/.checkov.yaml new file mode 100644 index 000000000..8962e2283 --- /dev/null +++ b/.checkov.yaml @@ -0,0 +1,48 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-21 16:53:29 +0000 (Mon, 21 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# C h e c k o v c o n f i g +# ============================================================================ # + +# https://github.com/bridgecrewio/checkov#configuration-using-a-config-file +# +# This is not well documented but the fields seem to be the same as: +# +# checkov --help +# +# See master template at: +# +# https://github.com/HariSekhon/Templates/blob/master/.checkov.yaml + +--- +compact: true +directory: + - . +skip-path: + - bash-tools + - pylib + - sql + - templates +docker-image: harisekhon/pytools +download-external-modules: true # without this gets lots of annoying warning lines such as '2022-02-22 16:14:40,180 [MainThread ] [WARNI] Failed to download module x/y/z:n.n.n' +framework: + - all +no-guide: true +output: cli +quiet: true +repo-id: HariSekhon/DevOps-Python-tools # what to report to Bridgecrew Cloud - without this gets annoying duplicate repos such as 'harisekhon_cli_repo/pytools' +skip-suppressions: true +soft-fail: true diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..312bf1020 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-23 23:30:14 +0000 (Sun, 23 Feb 2020) +# Original: H1 2016 (Circle CI 1.x) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# C i r c l e C I +# ============================================================================ # + +# Master Template with more advanced config: +# +# https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml + +# Reference: +# +# https://circleci.com/docs/2.0/configuration-reference + +version: 2.1 + +workflows: + version: 2 + workflow: + jobs: + - build + +jobs: + build: + docker: + - image: cimg/base:2024.12 + resource_class: small + steps: + # CLI is too old - config validate breaks in test - install new version to fix + # doesn't work - existing version is too old to update + #- run: circleci update + - run: | + curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash + - checkout + #- setup_remote_docker: + # version: 20.10.11 + - run: setup/ci_bootstrap.sh + - run: make init + - run: make + - run: make test diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 000000000..471dc55ba --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,31 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 16:55:36 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# C i r r u s C I +# ============================================================================ # + +# https://cirrus-ci.org/guide/writing-tasks/ + +container: + image: ubuntu:18.04 + +task: + env: + TMPDIR: /var/tmp + script: + - setup/ci_bootstrap.sh + - make init + - make ci test diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 000000000..d105ab57a --- /dev/null +++ b/.drone.yml @@ -0,0 +1,49 @@ +--- +# XXX: putting this separator further down with code causes a parsing bug in drone lint +# +# Author: Hari Sekhon +# Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# D r o n e C I +# ============================================================================ # + +# https://docs.drone.io/quickstart/cli/ +# +# https://docs.drone.io/cli/install/ +# +# brew install drone-cli +# +# cd to this directory +# +# drone exec [--pipeline default] [--include=thisstep] [--exclude=thatstep] + +kind: pipeline +type: docker +name: default + +steps: + - name: build + image: ubuntu:18.04 + #environment: + # DEBUG: 1 + commands: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test + +trigger: + branch: + - master diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..462f53206 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,85 @@ +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# http://EditorConfig.org + +# stop recursing upwards for other .editorconfig files +root = true + +# Unix-style newlines with a newline ending every file +[*] +indent_size = 4 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.go] +indent_size = 4 +indent_style = tab +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[Makefile] +indent_size = 4 +indent_style = tab +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[{*.md,*.hcl,*.tf,*.tfvars}] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.yml,*.yaml] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[.*] +indent_size = 4 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +# ============================================================================ # +# Older Stuff, don't think I use this any more +# ============================================================================ # + +# Matches multiple files with brace expansion notation +# Set default charset +#[*.{js,py}] +#charset = utf-8 + +# Indentation override for all JS under lib directory +#[lib/**.js] +#indent_style = space +#indent_size = 2 + +# Matches the exact files either package.json or .travis.yml +#[{package.json,.travis.yml}] +#indent_style = space +#indent_size = 2 + +#[*.xml] +#indent_style = space +#indent_size = 2 diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..e4488d51f --- /dev/null +++ b/.envrc @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# D i r E n v +# ============================================================================ # + +# https://direnv.net/man/direnv-stdlib.1.html + +# See Also: +# +# .envrc-aws +# .envrc-gcp +# .envrc-kubernetes + +# direnv stdlib - loads .envrc from parent dir up to / +# +# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. +# +# bypasses security authorization though - use with care +#source_up +# +# source_up must be loaded before set -u otherwise gets this error: +# +# direnv: loading .envrc +# /bin/bash: line 226: $1: unbound variable +# +# source_up causes this error is up .envrc is found in parent directories: +# +# direnv: No ancestor .envrc found + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +src="$(readlink -f "${BASH_SOURCE[0]}")" +srcdir="$(cd "$(dirname "$src")" && pwd)" + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +# Automatically install Pre-Commit Git hooks if not already present + +if ! type -P pre-commit &>/dev/null; then + if uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Homebrew..." + echo + brew install pre-commit + echo + elif type -P pip &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Pip..." + echo + pip install pre-commit + fi +fi + +if [ -f .pre-commit-config.yaml ] && + type -P pre-commit &>/dev/null && + git rev-parse --is-inside-work-tree &>/dev/null; then + hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" + if [ -L "$hook" ]; then + echo "Detected symlink hook: " + echo + ls -l "$hook" + echo + echo "Removing" + rm -f "$hook" + fi + if ! [ -f "$hook" ]; then + echo + echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." + echo + pre-commit install + fi +fi + +# ============================================================================ # +# D o c k e r C o m p o s e +# ============================================================================ # + +export COMPOSE_PROJECT_NAME="DevOps-Python-tools" + +# ============================================================================ # +# G i t H u b +# ============================================================================ # + +#export GITHUB_ORGANIZATION=HariSekhon + +# ============================================================================ # +# A n s i b l e +# ============================================================================ # + +# use the local repo's ansible.cfg rather than: +# +# $PWD/ansible.cfg +# ~/.ansible.cfg +# /etc/ansible/ansible.cfg +# +# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used +#export ANSIBLE_CONFIG="/path/to/ansible.cfg" + +# ============================================================================ # +# C l o u d f l a r e +# ============================================================================ # + +#export CLOUDFLARE_EMAIL=hari@... +#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens +#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module + +# export the variables for terraform +#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" +#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above + +# ============================================================================ # +# Load External Envrc Files If Present +# ============================================================================ # + +# XXX: safer to bring all these external .envrc inline if you're worried about changes +# to it bypassing 'direnv allow' authorization +load_if_exists(){ + # first arg is a path to a .envrc + # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes + # to pass the context name 'docker-desktop' to switch to + local envrc="$1" + shift + if ! [[ "$envrc" =~ ^/ ]]; then + envrc="$srcdir/$envrc" + fi + if [ -f "$envrc" ]; then + # prevent looping on symlinks to this .envrc if given + if [ "$(readlink "$envrc")" = "$src" ]; then + return + fi + echo + echo "Loading $envrc" + # shellcheck disable=SC1090,SC1091 + . "$envrc" "$@" + fi +} + +# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc +# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses +# to install to system Python) +#load_if_exists ~/.envrc + +# ============================================================================ # +# P y t h o n +# ============================================================================ # + + #.envrc-aws \ + #.envrc-gcp \ + #.envrc-terraform \ +# shellcheck disable=SC2043 +for envrc in \ + .envrc-python \ + ; do + load_if_exists "$envrc" +done + +# ============================================================================ # +# A W S +# ============================================================================ # + +if [[ "$PWD" =~ /aws/ ]]; then + load_if_exists .envrc-aws +fi + +# ============================================================================ # +# G C P +# ============================================================================ # + +if [[ "$PWD" =~ /gcp/ ]]; then + load_if_exists .envrc-gcp +fi + +# ============================================================================ # +# T e r r a f o r m +# ============================================================================ # + +if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then + load_if_exists .envrc-terraform +fi + +# ============================================================================ # +# K u b e r n e t e s +# ============================================================================ # + +if [ -f "$srcdir/.envrc-kubernetes" ]; then + load_if_exists .envrc-kubernetes docker-desktop +fi + +# ============================================================================ # +# . E n v +# ============================================================================ # + +echo +# read .env too +#dotenv + +load_if_exists .envrc.local diff --git a/.envrc-python b/.envrc-python new file mode 100644 index 000000000..6bc2d65b1 --- /dev/null +++ b/.envrc-python @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y t h o n D i r E n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + break + fi +done + +# read .env too +#dotenv diff --git a/.flake8 b/.flake8 index e31fb949f..de51ee708 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,32 @@ +# +# Author: Hari Sekhon +# Date: Mon Oct 21 15:57:10 2019 +0100 +# +# vim:ts=4:sts=4:sw=4:et +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# F l a k e 8 C o n f i g +# ============================================================================ # + +# https://flake8.pycqa.org/en/latest/user/configuration.html + [flake8] -ignore = E265,E402,F401 + max-line-length = 120 + +ignore = E265, + E402, + F401 + exclude = test*/* + max-complexity = 10 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..ad861088b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,38 @@ +# +# Author: Hari Sekhon +# Date: 2021-11-09 15:14:59 +0000 (Tue, 09 Nov 2021) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +# Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam + +# Tips: +# +# * includes changes under .github/ +# dir/* only matches first level file changes but doesn't recurse +# dir/ recurses +# +# - CODEOWNERS in base branch of PR determines review request +# - paths are case sensitive +# - last match wins, use * at top for overall owner then override with more specific teams + +#* @harisekhon # username or email address +#* @myorg/platform-engineering # team based is the way to go - team must have Write access to the repo regardless of if individuals have access +#* @myorg/devops +#k8s @myorg/devops @myorg/sre-team +#apps/ @myorg/developers +#apps/dir2 # ignores dir2 as no owner/team specified on this line +#src/ @myorg/developers +#docs/ docs@example.com +#.github/workflows @ci-cd-team diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 5f04c0a58..aa46b9847 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,3 +1,3 @@ Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell. -You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively. +You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) respectively. diff --git a/.github/workflows/*_centos.yaml.disabled b/.github/workflows/*_centos.yaml.disabled new file mode 100644 index 000000000..51d70819e --- /dev/null +++ b/.github/workflows/*_centos.yaml.disabled @@ -0,0 +1,124 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Docker Build (CentOS) + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo_tags: | + harisekhon/pytools:centos + ghcr.io/harisekhon/pytools:centos + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-centos + debug: ${{ github.event.inputs.debug }} + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml new file mode 100644 index 000000000..9bed93efb --- /dev/null +++ b/.github/workflows/alpine.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Alpine + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/alpine.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: alpine:latest + caches: apk pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml new file mode 100644 index 000000000..f6c4f5ea9 --- /dev/null +++ b/.github/workflows/alpine_3.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Alpine 3 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/alpine_3.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: alpine:3 + caches: apk pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos.yaml.disabled b/.github/workflows/centos.yaml.disabled new file mode 100644 index 000000000..420b88392 --- /dev/null +++ b/.github/workflows/centos.yaml.disabled @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: CentOS + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/centos.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: centos:latest + caches: yum pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos7.yaml.disabled b/.github/workflows/centos7.yaml.disabled new file mode 100644 index 000000000..ec764d288 --- /dev/null +++ b/.github/workflows/centos7.yaml.disabled @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: CentOS 7 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/centos7.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: centos:7 + caches: yum pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/centos8.yaml.disabled b/.github/workflows/centos8.yaml.disabled new file mode 100644 index 000000000..76c476664 --- /dev/null +++ b/.github/workflows/centos8.yaml.disabled @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: CentOS 8 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/centos8.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: centos:8 + caches: yum pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml new file mode 100644 index 000000000..209b3cef1 --- /dev/null +++ b/.github/workflows/checkov.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# C h e c k o v G i t H u b W o r k f l o w +# ============================================================================ # + +# Static analysis of Terraform code - publishes report to GitHub Security tab + +# https://github.com/bridgecrewio/checkov-action + +--- +name: Checkov + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +jobs: + checkov: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Checkov + uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml new file mode 100644 index 000000000..168d4914c --- /dev/null +++ b/.github/workflows/codeowners.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# C o d e O w n e r s +# ============================================================================ # + +--- +name: CodeOwners + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - CODEOWNERS + - .github/CODEOWNERS + pull_request: + branches: + - master + - main + paths: + - CODEOWNERS + - .github/CODEOWNERS + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Validate CODEOWNERS + uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..5e74ab863 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,63 @@ +--- +name: "CodeQL" + +on: + push: + branches: + - master + pull_request: + # The branches below must be a subset of the branches above + branches: + - master + schedule: + - cron: '37 15 * * 4' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: + - python + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # Command-line programs to run using the OS shell. + # https://git.io/JvXDl + + # If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml new file mode 100644 index 000000000..d84cb16b1 --- /dev/null +++ b/.github/workflows/debian.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:latest + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml new file mode 100644 index 000000000..e857f280f --- /dev/null +++ b/.github/workflows/debian_10.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 10 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_10.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:10 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml new file mode 100644 index 000000000..ad765f1ad --- /dev/null +++ b/.github/workflows/debian_11.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 11 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_11.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:11 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml new file mode 100644 index 000000000..dfbc4ec97 --- /dev/null +++ b/.github/workflows/debian_12.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 12 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_12.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:12 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled new file mode 100644 index 000000000..f3062b6f8 --- /dev/null +++ b/.github/workflows/debian_6.yaml.disabled @@ -0,0 +1,50 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 6 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:6 + # causes nodejs errors + #caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled new file mode 100644 index 000000000..1b0e00295 --- /dev/null +++ b/.github/workflows/debian_7.yaml.disabled @@ -0,0 +1,50 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 7 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:7 + # causes nodejs errors + #caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_8.yaml.disabled b/.github/workflows/debian_8.yaml.disabled new file mode 100644 index 000000000..b41ead7d3 --- /dev/null +++ b/.github/workflows/debian_8.yaml.disabled @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 8 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_8.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:8 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/debian_9.yaml.disabled b/.github/workflows/debian_9.yaml.disabled new file mode 100644 index 000000000..451348091 --- /dev/null +++ b/.github/workflows/debian_9.yaml.disabled @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Debian 9 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/debian_9.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: debian:9 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml new file mode 100644 index 000000000..7119d6852 --- /dev/null +++ b/.github/workflows/docker_pytools_alpine.yaml @@ -0,0 +1,124 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Docker Build (Alpine) + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo_tags: | + harisekhon/pytools:alpine + ghcr.io/harisekhon/pytools:alpine + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-alpine + debug: ${{ github.event.inputs.debug }} + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml new file mode 100644 index 000000000..24f8f21dd --- /dev/null +++ b/.github/workflows/docker_pytools_debian.yaml @@ -0,0 +1,124 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Docker Build (Debian) + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo_tags: | + harisekhon/pytools:debian + ghcr.io/harisekhon/pytools:debian + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-debian + debug: ${{ github.event.inputs.debug }} + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml new file mode 100644 index 000000000..ad1a5ed8d --- /dev/null +++ b/.github/workflows/docker_pytools_fedora.yaml @@ -0,0 +1,124 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Docker Build (Fedora) + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo_tags: | + harisekhon/pytools:fedora + ghcr.io/harisekhon/pytools:fedora + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-fedora + debug: ${{ github.event.inputs.debug }} + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml new file mode 100644 index 000000000..884917919 --- /dev/null +++ b/.github/workflows/docker_pytools_ubuntu.yaml @@ -0,0 +1,126 @@ +# +# Author: Hari Sekhon +# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Docker Build (Ubuntu) + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - .github/ + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master + with: + repo_tags: | + harisekhon/pytools:latest + harisekhon/pytools:ubuntu + ghcr.io/harisekhon/pytools:latest + ghcr.io/harisekhon/pytools:ubuntu + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-ubuntu + debug: ${{ github.event.inputs.debug }} + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml new file mode 100644 index 000000000..5a4f9a9b4 --- /dev/null +++ b/.github/workflows/fedora.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Fedora + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/fedora.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: fedora + caches: yum pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml new file mode 100644 index 000000000..6b228ca33 --- /dev/null +++ b/.github/workflows/fork-sync.yaml @@ -0,0 +1,48 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# F o r k S y n c +# ============================================================================ # + +# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch + +--- +name: Fork Sync + +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 */3 * * *' + +permissions: + contents: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + fork_sync: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == true + if: github.repository_owner != 'HariSekhon' + name: Fork Sync + uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml new file mode 100644 index 000000000..c74fa2a0b --- /dev/null +++ b/.github/workflows/fork-update-pr.yaml @@ -0,0 +1,51 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# F o r k U p d a t e P R +# ============================================================================ # + +# For a fork of the original repo, activate to keep its branches up to date via Pull Requests +# +# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date + +--- +name: Fork Update PR + +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 10 * * 1' + +permissions: + contents: write + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + fork_update_pr: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == true + if: github.repository_owner != 'HariSekhon' + name: Fork Update PR + uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ghcr_python_ubuntu.yaml.disabled b/.github/workflows/ghcr_python_ubuntu.yaml.disabled new file mode 100644 index 000000000..c4503bc13 --- /dev/null +++ b/.github/workflows/ghcr_python_ubuntu.yaml.disabled @@ -0,0 +1,36 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-09 18:07:10 +0000 (Wed, 09 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: GHCR Build (Ubuntu) + +on: + push: + branches: + - master + - main + workflow_dispatch: + +jobs: + docker_build: + name: Docker Build + uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build_ghcr.yaml@master + with: + image: pytools + tags: ubuntu latest + dockerfile-repo: HariSekhon/Dockerfiles + context: Dockerfiles/devops-python-tools-ubuntu + if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker' + permissions: + contents: read + packages: write diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml new file mode 100644 index 000000000..1b0fa15b4 --- /dev/null +++ b/.github/workflows/grype.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G r y p e +# ============================================================================ # + +--- +name: Grype + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + Grype: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Grype + uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml new file mode 100644 index 000000000..83fa5ef83 --- /dev/null +++ b/.github/workflows/json.yaml @@ -0,0 +1,58 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# J S O N +# ============================================================================ # + +# Validate any JSON files found in the repo + +--- +name: JSON + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.json' + - .github/workflows/json.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.json' + - .github/workflows/json.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + contents: read + +jobs: + check_json: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Check JSON + uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml new file mode 100644 index 000000000..c389f5df0 --- /dev/null +++ b/.github/workflows/kics.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# K i c s +# ============================================================================ # + +--- +name: Kics + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + kics: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Kics + uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml new file mode 100644 index 000000000..438dd184e --- /dev/null +++ b/.github/workflows/mac.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Mac + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/alpine.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + runs-on: macos-latest + caches: brew pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml new file mode 100644 index 000000000..cc392b105 --- /dev/null +++ b/.github/workflows/mac_11.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Mac 11 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/alpine.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + runs-on: macos-11 + caches: brew pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml new file mode 100644 index 000000000..0bcd6e6fd --- /dev/null +++ b/.github/workflows/mac_12.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Mac 12 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/alpine.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + runs-on: macos-12 + caches: brew pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml new file mode 100644 index 000000000..72e301ba0 --- /dev/null +++ b/.github/workflows/markdown.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# M a r k D o w n +# ============================================================================ # + +--- +name: Markdown + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + Markdown: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Markdown + uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master diff --git a/.github/workflows/pypy2.yaml.disabled b/.github/workflows/pypy2.yaml.disabled new file mode 100644 index 000000000..01080bcc8 --- /dev/null +++ b/.github/workflows/pypy2.yaml.disabled @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: PyPy 2 + +on: + push: + branches: + - master + - main + paths: + - '**/*.py' + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: PyPy2 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: pypy2 + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/pypy3.yaml.disabled b/.github/workflows/pypy3.yaml.disabled new file mode 100644 index 000000000..101b36751 --- /dev/null +++ b/.github/workflows/pypy3.yaml.disabled @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: PyPy 3 + +on: + push: + branches: + - master + - main + paths: + - '**/*.py' + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: PyPy3 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: pypy3 + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python2.7.yaml.disabled b/.github/workflows/python2.7.yaml.disabled new file mode 100644 index 000000000..1813931e7 --- /dev/null +++ b/.github/workflows/python2.7.yaml.disabled @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 2.7 + +on: + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python2.7.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python2.7.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 2.7 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: 2.7 + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml new file mode 100644 index 000000000..8ec3008b9 --- /dev/null +++ b/.github/workflows/python3.10.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.10 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.10.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.10.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.10 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: "3.10" + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml new file mode 100644 index 000000000..0f27b0076 --- /dev/null +++ b/.github/workflows/python3.11.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.11 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.11.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.11 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: "3.11" + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.6.yaml.disabled b/.github/workflows/python3.6.yaml.disabled new file mode 100644 index 000000000..91bd3e673 --- /dev/null +++ b/.github/workflows/python3.6.yaml.disabled @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.6 + +on: + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.6.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.6.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.6 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: 3.6 + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml new file mode 100644 index 000000000..73a0045c7 --- /dev/null +++ b/.github/workflows/python3.7.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.7 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.7.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.7.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.7 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: 3.7 + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml new file mode 100644 index 000000000..abcbc4c68 --- /dev/null +++ b/.github/workflows/python3.8.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.8 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.8.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.8.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.8 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: 3.8 + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml new file mode 100644 index 000000000..1ce7eb238 --- /dev/null +++ b/.github/workflows/python3.9.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Python 3.9 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.9.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.py' + - pylib + - requirements.txt + - .github/workflows/python3.9.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Python 3.9 + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + python-version: 3.9 + caches: apt pip + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml new file mode 100644 index 000000000..c2f614222 --- /dev/null +++ b/.github/workflows/semgrep-cloud.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S e m g r e p C l o u d W o r k f l o w +# ============================================================================ # + +# Logs results to https://semgrep.dev/ + +--- +name: Semgrep Cloud + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + semgrep: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Semgrep Cloud + uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master + secrets: + SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml new file mode 100644 index 000000000..04a3cf197 --- /dev/null +++ b/.github/workflows/semgrep.yaml @@ -0,0 +1,64 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S e m g r e p G i t H u b W o r k f l o w +# ============================================================================ # + +# Generates code scanning alerts in GitHub's Security tab -> Code scanning alerts + +# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions + +--- +name: Semgrep + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + semgrep: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Semgrep GitHub Security Tab + uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml new file mode 100644 index 000000000..246087bd4 --- /dev/null +++ b/.github/workflows/shellcheck.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S h e l l C h e c k +# ============================================================================ # + +# Validate any shell scripts found in the repo + +--- +name: ShellCheck + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.sh' + - .github/workflows/shellcheck.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.sh' + - .github/workflows/shellcheck.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + shellcheck: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: ShellCheck + uses: HariSekhon/GitHub-Actions/.github/workflows/shellcheck.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 000000000..95b2571ca --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# T r i v y +# ============================================================================ # + +# Scan files in the local repo + +--- +name: Trivy + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + trivy: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Trivy + uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml new file mode 100644 index 000000000..fe863f875 --- /dev/null +++ b/.github/workflows/ubuntu.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:latest + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_14.04.yaml.disabled b/.github/workflows/ubuntu_14.04.yaml.disabled new file mode 100644 index 000000000..1442d675d --- /dev/null +++ b/.github/workflows/ubuntu_14.04.yaml.disabled @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 14.04 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_14.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:14.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_16.04.yaml.disabled b/.github/workflows/ubuntu_16.04.yaml.disabled new file mode 100644 index 000000000..a9655835d --- /dev/null +++ b/.github/workflows/ubuntu_16.04.yaml.disabled @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 16.04 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_16.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:16.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml.disabled new file mode 100644 index 000000000..4b510ba1e --- /dev/null +++ b/.github/workflows/ubuntu_18.04.yaml.disabled @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 18.04 + +on: + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_18.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:18.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml new file mode 100644 index 000000000..47d0512ac --- /dev/null +++ b/.github/workflows/ubuntu_20.04.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 20.04 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_20.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:20.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml new file mode 100644 index 000000000..ba8a549b1 --- /dev/null +++ b/.github/workflows/ubuntu_22.04.yaml @@ -0,0 +1,85 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: Ubuntu 22.04 + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_22.04.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + container: ubuntu:22.04 + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml new file mode 100644 index 000000000..0037e4ba3 --- /dev/null +++ b/.github/workflows/ubuntu_github.yaml @@ -0,0 +1,84 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +--- +name: GitHub Actions Ubuntu + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + paths-ignore: + - '**/*.md' + - '.github/workflows/*' + - '!.github/workflows/ubuntu_github.yaml' + - '**/Dockerfile' + - '**/Jenkinsfile' + - '**/.envrc*' + - .checkov.yaml + - .dockerignore + - .gcloudignore + - .editorconfig + - '.gitconfig*' + - .gitignore + - .grype.yaml + - .hound.yml + - .terraformignore + - Jenkinsfile + - .appveyor.yml + - .buildkite/pipeline.yml + - .circleci/config.yml + - .cirrus.yml + - .concourse.yml + - .drone.yml + - .gitlab-ci.yml + - .gocd.yml + - .scrutinizer.yml + - .semaphore/semaphore.yml + - .travis.yml + - .werckerignore + - azure-pipelines.yml + - bitbucket-pipelines.yml + - buddy.yml + - buildspec.yml + - cloudbuild.yaml + - codefresh.yml + - codeship.yml + - shippable.yml + - wercker.yml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 7 * * *' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Build + uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master + with: + caches: apt pip cpanm + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml new file mode 100644 index 000000000..ab6ab9638 --- /dev/null +++ b/.github/workflows/validate.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# V a l i d a t i o n +# ============================================================================ # + +# Run all custom validations against files in the repo + +--- +name: Validation + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + #schedule: + # - cron: '0 0 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Validate + uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml new file mode 100644 index 000000000..3f6265a6a --- /dev/null +++ b/.github/workflows/xml.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# X M L +# ============================================================================ # + +# Validate any XML files found in the repo + +--- +name: XML + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.xml' + - .github/workflows/xml.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.xml' + - .github/workflows/xml.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + contents: read + +jobs: + check_xml: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Check XML + uses: HariSekhon/GitHub-Actions/.github/workflows/xml.yaml@master diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml new file mode 100644 index 000000000..f2a4ca129 --- /dev/null +++ b/.github/workflows/yaml.yaml @@ -0,0 +1,64 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# Y A M L +# ============================================================================ # + +# Validate any YAML files found in the repo + +--- +name: YAML + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.yml' + - '**/*.yaml' + - .github/workflows/yaml.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.yml' + - '**/*.yaml' + - .github/workflows/yaml.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check_yaml: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Check YAML + uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..625220b32 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,30 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: Sun Feb 23 19:02:10 2020 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G i t L a b C I +# ============================================================================ # + +# https://docs.gitlab.com/ee/ci/yaml/README.html + +#include: '.gitlab/*.y*ml' + +image: ubuntu:18.04 + +job: + before_script: + - setup/ci_bootstrap.sh + script: + - make init && make ci test diff --git a/.gitmodules b/.gitmodules index 37537e726..30fa60a67 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,16 @@ [submodule "pylib"] path = pylib - url = https://github.com/harisekhon/pylib + url = https://github.com/HariSekhon/pylib branch = master [submodule "bash-tools"] path = bash-tools - url = https://github.com/harisekhon/bash-tools + url = https://github.com/HariSekhon/DevOps-Bash-tools + branch = master +[submodule "sql"] + path = sql + url = https://github.com/HariSekhon/SQL-scripts + branch = master +[submodule "templates"] + path = templates + url = https://github.com/HariSekhon/Templates branch = master diff --git a/.ipython-notebook-pyspark.00-pyspark-setup.py b/.ipython-notebook-pyspark.00-pyspark-setup.py index 4dde897da..1de53b3fb 100644 --- a/.ipython-notebook-pyspark.00-pyspark-setup.py +++ b/.ipython-notebook-pyspark.00-pyspark-setup.py @@ -24,4 +24,4 @@ sys.path.insert(0, os.path.join(spark_home, 'python')) for lib in glob.glob(os.path.join(spark_home, 'python/lib/py4j-*-src.zip')): sys.path.insert(0, lib) -execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) +execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) # pylint: disable=undefined-variable diff --git a/.mdl.rb b/.mdl.rb new file mode 100644 index 000000000..f8f9b004c --- /dev/null +++ b/.mdl.rb @@ -0,0 +1,30 @@ +#!/usr/bin/env ruby +# vim:ts=4:sts=4:sw=4:et:filetype=ruby +# +# Author: Hari Sekhon +# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +all +#exclude_rule 'MD001' +#exclude_rule 'MD003' +#exclude_rule 'MD005' +exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af +#exclude_rule 'MD012' +exclude_rule 'MD013' # long lines cannot be split if they are URLs +#exclude_rule 'MD022' +#exclude_rule 'MD025' +#exclude_rule 'MD031' +#exclude_rule 'MD032' +exclude_rule 'MD033' # inline HTML is important for formatting +exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines +#exclude_rule 'MD039' +#exclude_rule 'MD056' diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 000000000..27e5b6895 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,5 @@ +mdlrc_dir = File.expand_path('..', __FILE__) + +style_file = File.join(mdlrc_dir, '.mdl.rb') + +style style_file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..787fb79ea --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,74 @@ +# +# Author: Hari Sekhon +# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024) +# +# vim:ts=2:sts=2:sw=2:et +# +# https///github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +--- +fail_fast: false +#exclude: *.tmp$ + +repos: + + # will accept anything that 'git clone' understands + # this means you can set this to a local git repo to develop your own hook repos interactively + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + # Common errors + #- id: end-of-file-fixer # ruins .gitignore Icon\r + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + #- id: forbid-new-submodules + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + + # rewrites python files with useless changes like changing single quotes to double quotes + #- repo: https://github.com/psf/black + # rev: 24.8.0 + # hooks: + # - id: black + + # Git secrets Leaks + - repo: https://github.com/awslabs/git-secrets.git + # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error: + # + # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file + # + rev: 5357e18 + hooks: + - id: git-secrets + + - repo: https://github.com/markdownlint/markdownlint + rev: v0.12.0 + hooks: + - id: markdownlint + name: Markdownlint + description: Run markdownlint on your Markdown files + entry: mdl + args: [-s, .mdl.rb] + language: ruby + files: \.(md|mdown|markdown)$ diff --git a/.pylintrc b/.pylintrc index 65e97ab22..2006619da 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,378 +1,655 @@ -[MASTER] +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y L i n t C o n f i g +# ============================================================================ # + +# pylint --generate-rcfile >> .pylintrc + +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= -# Specify a configuration file. -#rcfile= +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 -# Pickle collected data for later comparisons. -persistent=yes +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 -# List of plugins (as comma separated values of python modules names) to load, +# List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= -# Use multiple processes to speed up Pylint. -jobs=1 +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. -optimize-ast=no +[BASIC] -[MESSAGES CONTROL] +# Naming style matching correct argument names. +argument-naming-style=snake_case -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= +# Naming style matching correct attribute names. +attr-naming-style=snake_case -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111 +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata -[REPORTS] +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text +# Naming style matching correct class attribute names. +class-attribute-naming-style=any -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= -# Tells whether to display a full report or only the messages -reports=yes +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= +# Naming style matching correct class names. +class-naming-style=PascalCase +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= -[BASIC] +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,input +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +# Naming style matching correct module names. +module-naming-style=snake_case -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Naming style matching correct variable names. +variable-naming-style=snake_case -# Regular expression matching correct constant names -const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$ +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ +[CLASSES] -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ +[DESIGN] -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of arguments for function / method. +max-args=5 -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +# Maximum number of attributes for a class (see R0902). +max-attributes=7 -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ +# Maximum number of branch for function / method body. +max-branches=12 -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ +# Maximum number of locals for function / method body. +max-locals=15 -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 +# Maximum number of parents for a class (see R0901). +max-parents=7 +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 -[ELIF] +# Maximum number of return / yield for function / method body. +max-returns=6 -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] -# Maximum number of characters on a single line. -max-line-length=120 +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator -# Maximum number of lines in a module -max-module-lines=1000 +[IMPORTS] -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= [LOGGING] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + # Logging modules to check that the string format arguments are in logging -# function parameter format +# function parameter format. logging-modules=logging +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + missing-class-docstring, + missing-function-docstring, + super-with-arguments, + consider-using-f-string + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO +notes=FIXME, + XXX, + TODO +# Regular expression of note tags to take in consideration. +notes-rgx= -[SIMILARITIES] -# Minimum lines number of a similarity. -min-similarity-lines=4 +[REFACTORING] -# Ignore comments when computing similarities. -ignore-comments=yes +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 -# Ignore docstrings when computing similarities. -ignore-docstrings=yes +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error -# Ignore imports when computing similarities. -ignore-imports=no +[REPORTS] -[SPELLING] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= -# List of comma separated words that should not be checked. -spelling-ignore-words= +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= +# Tells whether to display a full report or only the messages. +reports=no -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no +# Activate the evaluation score. +score=yes -[TYPECHECK] +[SIMILARITIES] -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes +# Comments are removed from the similarity computation +ignore-comments=yes -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= +# Docstrings are removed from the similarity computation +ignore-docstrings=yes -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). This supports can work -# with qualified names. -ignored-classes= +# Imports are removed from the similarity computation +ignore-imports=yes -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= +# Signatures are removed from the similarity computation +ignore-signatures=yes +# Minimum lines number of a similarity. +min-similarity-lines=4 -[VARIABLES] -# Tells whether we should check for unused import in __init__ files. -init-import=no +[SPELLING] -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: +# List of comma separated words that should not be checked. +spelling-ignore-words= -[CLASSES] +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs +[STRING] -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no -[DESIGN] -# Maximum number of arguments for function / method -max-args=5 +[TYPECHECK] -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager -# Maximum number of locals for function / method body -max-locals=15 +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= -# Maximum number of return / yield for function / method body -max-returns=6 +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes -# Maximum number of branch for function / method body -max-branches=12 +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes -# Maximum number of statements in function / method body -max-statements=50 +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init -# Maximum number of parents for a class (see R0901). -max-parents=7 +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace -# Maximum number of attributes for a class (see R0902). -max-attributes=7 +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin +# List of decorators that change the signature of a decorated function. +signature-mutators= -[IMPORTS] -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec +[VARIABLES] -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= +# List of names allowed to shadow builtins +allowed-redefined-builtins= +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb -[EXCEPTIONS] +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml new file mode 100644 index 000000000..59aa80836 --- /dev/null +++ b/.semaphore/semaphore.yml @@ -0,0 +1,124 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S e m a p h o r e C I +# ============================================================================ # + +# https://docs.semaphoreci.com/reference/pipeline-yaml-reference/ + +version: v1.0 +name: DevOps-Python-tools +agent: + # https://docs.semaphoreci.com/reference/machine-types#linux + machine: + type: e1-standard-2 + os_image: ubuntu2004 +execution_time_limit: + hours: 3 +blocks: + - name: Linux build + run: + when: "branch = 'master' AND change_in('/', {exclude: ['**/*.md']})" + #execution_time_limit: + # hours: 2 + task: + #env_vars: + # $PATH selects /usr/bin/python and /usr/local/bin/pip which are mismatched versions of Python + #- name: PYTHON + # value: python3 + #- name: PIP + # value: pip3 + prologue: + commands: + - cache restore + # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server + # + # causes error: + # + # Not replacing deleted config file /etc/ssh/sshd_config + # + #- sudo rm -f /etc/ssh/sshd_config + - export DEBIAN_FRONTEND=noninteractive + - sudo -E apt-get update + - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew" + - sudo dpkg --configure -a --force-confmiss --force-confnew + #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections + #- sudo dpkg --configure -a --force-confdef --force-confold + #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" + - sudo apt-get install -y openssh-server + # each job is separate and could be run on a separate machine so all steps must be together + jobs: + - name: build + commands: + - checkout + - setup/ci_bootstrap.sh + - make init + - make ci + - make test + epilogue: + commands: + - cache store + - name: Mac build + run: + when: "branch = 'master'" + task: + # because otherwise on Mac it uses /usr/bin/python (2.7) but /usr/local/bin/pip (python 3.8) + #env_vars: + # to match /usr/local/bin/pip version from $PATH + #- name: PYTHON + # value: python3 + # must be quoted to force string, otherwise pipeline fails to run with this parsing error: + # Error: [{"Type mismatch. Expected String but got Integer.", "#/blocks/1/task/env_vars/1/value"}] + #- name: DEBUG + # value: "1" + agent: + # https://docs.semaphoreci.com/reference/machine-types#macos + machine: + type: a1-standard-4 + os_image: macos-xcode15 + prologue: + commands: + - cache restore + # fix for: + # pip is configured with locations that require TLS/SSL, however the ssl module in Python is not available. + - brew install openssl + - brew reinstall python + - brew reinstall wget + # avoid Mac SSL errors: + # + # ERROR: Loading command: install (LoadError) + # dlopen(/Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle, 9): Library not loaded: /usr/local/opt/openssl/lib/libssl.1.0.0.dylib + # Referenced from: /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle + # Reason: image not found - /Users/semaphore/.rbenv/versions/2.5.1/lib/ruby/2.5.0/x86_64-darwin18/openssl.bundle + # ERROR: While executing gem ... (NoMethodError) + # undefined method `invoke_with_build_args' for nil:NilClass# + # + - rbenv global system + # also considered this: + # - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done + # + # fix for python vs pip version mismatch + - ln -svf -- /usr/local/bin/python3 /usr/local/bin/python + jobs: + - name: build + commands: + - checkout + - make init + - make ci + - make test + epilogue: + commands: + - cache store diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 000000000..a4496869e --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1 @@ +sonar.host.url=https://sonarcloud.io diff --git a/.travis.yml b/.travis.yml index 0ef2d390c..47f2f80b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,61 +3,137 @@ # Author: Hari Sekhon # Date: 2014-11-29 01:02:47 +0000 (Sat, 29 Nov 2014) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# T r a v i s C I +# ============================================================================ # + +# https://docs.travis-ci.com/user/customizing-the-build/ + +--- version: ~> 1.0 +# ============================ # Mac Python 2.7 not available - see https://travis-ci.org/HariSekhon/pylib/jobs/584226228 #os: # - linux # - osx +# ================================================= +# https://docs.travis-ci.com/user/languages/python/ +python: + - "2.7" + #- "3.4" supported by pip as of March 2019 + - "3.5" + - "3.6" + - "3.7" + - "3.8" + - "pypy" # currently Python 2.7.13, PyPy 7.1.1 + - "pypy3" # currently Python 3.6.1, PyPy 7.1.1-beta0 + +# ============================================= +# https://docs.travis-ci.com/user/build-matrix/ +# https://docs.travis-ci.com/user/build-stages/matrix-expansion/ matrix: + fast_finish: true include: + # numpy has gone 2.7+ only now, so had to drop Python 2.6 support - os: linux language: python - python: - # - "2.6" - - "2.7" - # MySQL in lib doesn't build from pip in Python 3 - # - "3.2" - # - "3.3" - # - "3.4" - # - "3.5" - # python-krbV fails to compile on PyPy - # - "pypy" - # - "pypy3" - # workaround is to use generic and install to system python + python: "2.7" + - os: osx - language: generic + language: generic # workaround since Mac doesn't have Python support yet, so install to system Python + # https://docs.travis-ci.com/user/reference/osx/ + # macOS 10.15.7 - otherwise defaults to Mac macOS 10.13 with xcode9.4 otherwise - and HomeBrew update takes 50 minutes until the build times out :-/ + osx_image: xcode12.2 + + - os: linux + language: python + python: "3.5" + + - os: linux + language: python + python: "3.6" + + - os: linux + language: python + python: "3.7" -dist: trusty + - os: linux + language: python + python: "3.8" -sudo: required + # python-krbV fails to compile on PyPy + # + # psutil doesn't build: + # + # RuntimeError: broken / incompatible Python implementation, see: https://github.com/giampaolo/psutil/issues/1659 + # + #- os: linux + # language: python + # python: "pypy" + - os: linux + language: python + python: "pypy3" + + # =================================================================================================== + # https://docs.travis-ci.com/user/multi-os/#allowing-failures-on-jobs-running-on-one-operating-system + allow_failures: + - python: "pypy" + - python: "pypy3" + +# ======================================= +# https://docs.travis-ci.com/user/docker/ +services: + - docker + +# ====================================================== +# https://docs.travis-ci.com/user/environment-variables/ env: # - DEBUG=1 - DOCKER_COMPOSE_VERSION=1.16.1 PYTHONUNBUFFERED=1 +# ============================================== +# https://docs.travis-ci.com/user/notifications/ notifications: email: false -branches: - only: - - master +# ================================================================================= +# https://docs.travis-ci.com/user/customizing-the-build/#building-specific-branches +# https://docs.travis-ci.com/user/conditional-builds-stages-jobs +#branches: +# only: +# - master -cache: pip +# ======================================== +# https://docs.travis-ci.com/user/caching/ -services: - - docker +before_cache: + #- rm -f $HOME/.cache/pip/log/debug.log + # XXX: cache breaks pypy builds, so clear it + - rm -f $HOME/.cache/pip + +cache: + - pip + - directories: + - $HOME/.cache + - $HOME/.cpan + - $HOME/.cpanm + - $HOME/.gem + +# ============================================== +# https://docs.travis-ci.com/user/job-lifecycle/ # avoid package checksum mismatches when installing packages before_install: diff --git a/.yamllint b/.yamllint index b5e6f8448..e29153b7f 100644 --- a/.yamllint +++ b/.yamllint @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # --- diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..d582bd112 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,88 @@ +// vim:ts=4:sts=4:sw=4:et:filetype=groovy:syntax=groovy +// +// Author: Hari Sekhon +// Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017) +// +// https://github.com/HariSekhon/DevOps-Python-tools +// +// License: see accompanying Hari Sekhon LICENSE file +// +// If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +// +// https://www.linkedin.com/in/HariSekhon +// + +// ========================================================================== // +// J e n k i n s P i p e l i n e +// ========================================================================== // + +// Epic Jenkinsfile template: +// +// https://github.com/HariSekhon/Templates/blob/master/Jenkinsfile + + +// Official Documentation: +// +// https://jenkins.io/doc/book/pipeline/syntax/ +// +// https://www.jenkins.io/doc/pipeline/steps/ +// +// https://www.jenkins.io/doc/pipeline/steps/workflow-basic-steps/ + + +pipeline { + // to run on Docker or Kubernetes, see the master Jenkinsfile template listed at the top + agent any + + options { + timestamps() + + timeout(time: 2, unit: 'HOURS') + } + + triggers { + cron('H 10 * * 1-5') + pollSCM('H/2 * * * *') + } + + stages { + stage ('Checkout') { + steps { + checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/HariSekhon/DevOps-Python-tools']]]) + } + } + + stage('Build') { + steps { + echo "Running ${env.JOB_NAME} Build ${env.BUILD_ID} on ${env.JENKINS_URL}" + echo 'Building...' + timeout(time: 10, unit: 'MINUTES') { + retry(3) { +// sh 'apt update -q' +// sh 'apt install -qy make' +// sh 'make init' + sh """ + setup/ci_bootstrap.sh && + make init + """ + } + } + timeout(time: 180, unit: 'MINUTES') { + sh 'make ci' + } + } + } + + stage('Test') { + options { + retry(2) + } + steps { + echo 'Testing...' + timeout(time: 120, unit: 'MINUTES') { + sh 'make test' + } + } + } + } +} diff --git a/LICENSE b/LICENSE index 03b527051..4860c1cf5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,45 +1,7 @@ -======================================= -HARI SEKHON LICENSE Revision 2013112300 -======================================= +Copyright 2015 Hari Sekhon -Copyright (c) 2006 onwards, Hari Sekhon -All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -Redistribution and use in source and binary forms, with or without -modification, is permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -3. All advertising materials mentioning features or use of this software - must display the following acknowledgement: - This product includes software developed by Hari Sekhon. -4. Neither the name Hari Sekhon nor any affiliates may be used to endorse or - promote products derived from this software without specific prior written - permission. -5. Modifications may be released to the public only with prior written permission - from Hari Sekhon. Forking on GitHub is permitted for the purpose of creating - patch pull requests back to the original repository. -6. Private modifications may be made to suit requirements, but any modifications - to this work, whether publicly disclosed or not, must be sent back to - Hari Sekhon via GitHub (https://github.com/harisekhon/devops-python-tools) or - LinkedIn (https://www.linkedin.com/in/harisekhon) - and must come under this same license. Any such modifications may be - reincorporated for the improvement of this software. -7. This work may not be sold without prior written permission from Hari Sekhon -8. This license may change at any time and the latest revision supersedes - all prior revisions. -9. Alternative licensing must be agreed in writing with Hari Sekhon prior - to public availability. +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THIS SOFTWARE IS PROVIDED BY Hari Sekhon ''AS IS'' AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL Hari Sekhon OR ANY AFFILIATED BODY BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile old mode 100755 new mode 100644 index f77f6f741..1d98ac524 --- a/Makefile +++ b/Makefile @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules to non-system python otherwise they're not found by programs. @@ -15,20 +15,27 @@ # =================== # bootstrap commands: +# setup/bootstrap.sh +# +# OR +# # Alpine: # -# apk add --no-cache git $(MAKE) && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE) - +# apk add --no-cache git make && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make +# # Debian / Ubuntu: # -# apt-get update && apt-get install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE) - +# apt-get update && apt-get install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make +# # RHEL / CentOS: # -# yum install -y $(MAKE) git && git clone https://github.com/harisekhon/devops-python-tools && cd pytools && $(MAKE) +# yum install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make # =================== +# would fail bootstrapping on Alpine +#SHELL := /usr/bin/env bash + ifneq ("$(wildcard bash-tools/Makefile.in)", "") include bash-tools/Makefile.in endif @@ -46,25 +53,58 @@ ifndef SKIP_PARQUET endif .PHONY: build -build: +build: init @echo ========================= @echo DevOps Python Tools Build @echo ========================= + @$(MAKE) git-summary + @echo + # defer via external sub-call, otherwise will result in error like + # make: *** No rule to make target 'python-version', needed by 'build'. Stop. + @$(MAKE) python-version - $(MAKE) init if [ -z "$(CPANM)" ]; then make; exit $$?; fi $(MAKE) system-packages-python if type apk 2>/dev/null; then $(MAKE) apk-packages-extra; fi if type apt-get 2>/dev/null; then $(MAKE) apt-packages-extra; fi + $(MAKE) python .PHONY: init init: git submodule update --init --recursive -.PHONY: python -python: +# Magic to build the dependencies for only given program(s) +# +# dependency of same % stem prefix checks for a matching .py file to consider it a valid target +# +# doesn't work +#.PHONY: all +#.PHONY: %.pyc +# TODO: doesn't work, says nothing to be done even when .pyc isn't present, and allows make anonymize22.py which breaks +#%.py: %.pyc +# @$(MAKE) $@c +%.pyc:: %.py + @# this utility script supports taking .pyc or .pyo names and still does the right thing + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + python -m py_compile $< && \ + echo && \ + echo Generated $@ +%.pyo:: %.py + @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \ + python -O -m py_compile $< && \ + echo && \ + echo Generated $@ + +.PHONY: pylib +pylib: + @$(MAKE) python-version cd pylib && $(MAKE) + +.PHONY: python +python: pylib + # defer via external sub-call, otherwise will result in error like + # make: *** No rule to make target 'python-version', needed by 'build'. Stop. @# don't pull parquet tools in to docker image by default, will bloat it @# can fetch separately by running 'make parquet-tools' if you really want to @if [ -f /.dockerenv -o -n "$(SKIP_PARQUET)" ]; then \ @@ -76,19 +116,20 @@ python: fi @# only install pip packages not installed via system packages - @#$(SUDO_PIP) pip install --upgrade -r requirements.txt - @#$(SUDO_PIP) pip install -r requirements.txt - @PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt + @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt + @#$(SUDO_PIP) $(PIP) install -r requirements.txt + @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt - @# python-krbV dependency doesn't build on Mac any more and is unmaintained + @# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3 @# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit - @bash-tools/python_pip_install.sh snakebite[kerberos] || : + PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || : # Python >= 3.4 - try but accept failure in case we're not on the right version of Python - @if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi + @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3"; fi + PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3" || : @# for impyla - @#$(SUDO_PIP) pip install --upgrade setuptools || : + @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || : @# @# snappy may fail to install on Mac not finding snappy-c.h - workaround: @# @@ -99,21 +140,21 @@ python: @# /usr/local/include/snappy-c.h @# @# sudo su - @# LD_RUN_PATH=/usr/local/include pip install snappy + @# LD_RUN_PATH=/usr/local/include $(PIP) install snappy @# - @#$(SUDO_PIP) pip install --upgrade -r requirements.txt + @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt @# for ipython-notebook-pyspark.py - @#$(SUDO_PIP) pip install jinja2 + @#$(SUDO_PIP) PIP=$(PIP) install jinja2 @# HiveServer2 - @#$(SUDO_PIP) pip install pyhs2 + @#$(SUDO_PIP) $(PIP) install pyhs2 @# Impala - @#$(SUDO_PIP) pip install impyla + @#$(SUDO_PIP) $(PIP) install impyla @# must downgrade happybase library to work on Python 2.6 - @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi + @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi @# Python >= 2.7 - won't build on 2.6, handle separately and accept failure - @bash-tools/python_pip_install.sh "ipython[notebook]" || : + @PIP=$(PIP) bash-tools/python/python_pip_install.sh "ipython[notebook]" || : @echo $(MAKE) pycompile @echo @@ -124,7 +165,7 @@ python: .PHONY: parquet-tools parquet-tools: - @BIN='.' bash-tools/setup/install_parquet-tools.sh + @BIN='.' bash-tools/install/install_parquet-tools.sh .PHONY: apk-packages-extra apk-packages-extra: @@ -134,7 +175,7 @@ apk-packages-extra: .PHONY: apt-packages-extra apt-packages-extra: - if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk; fi + if [ -z "$(NOJAVA)" ]; then which java || bash-tools/packages/apt_install_packages.sh default-jdk; fi # for validate_multimedia.py # available in Alpine 2.6, 2.7 and 3.x @@ -148,8 +189,8 @@ apk-packages-multimedia: # Debian 9 Stretch onwards, not available in Debian 8 Jessie .PHONY: apt-packages-multimedia apt-packages-multimedia: - $(SUDO) apt-get update - $(SUDO) apt-get install -y --no-install-recommends ffmpeg + $(SUDO) apt-get update -o DPkg::Lock::Timeout=1200 + $(SUDO) apt-get install -o DPkg::Lock::Timeout=1200 -y --no-install-recommends ffmpeg # for validate_multimedia.py .PHONY: yum-packages-multimedia @@ -159,9 +200,10 @@ yum-packages-multimedia: .PHONY: jython jython: - if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi - if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi - if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + @#if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi + @#if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi + @#if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi + bash-tools/packages/install_packages.sh wget expect sh jython_install.sh .PHONY: test-lib @@ -169,17 +211,13 @@ test-lib: cd pylib && $(MAKE) test .PHONY: test -test: test-lib +#test: test-lib +test: tests/all.sh .PHONY: basic-test basic-test: test-lib - bash-tools/check_all.sh - -.PHONY: test2 -test2: - cd pylib && $(MAKE) test2 - tests/all.sh + bash-tools/checks/check_all.sh .PHONY: install install: build diff --git a/README.md b/README.md index c52a0d279..5e1fe9bca 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,182 @@ -Hari Sekhon - DevOps Python Tools -================================= -[![Build Status](https://travis-ci.org/HariSekhon/DevOps-Python-tools.svg?branch=master)](https://travis-ci.org/HariSekhon/DevOps-Python-tools) -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/f7af72140c3b408b9659207ced17544f)](https://www.codacy.com/app/harisekhon/devops-python-tools) -[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/stargazers) -[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools.svg)](https://github.com/harisekhon/devops-python-tools/network) +# Hari Sekhon - DevOps Python Tools + +[![GitHub stars](https://img.shields.io/github/stars/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/harisekhon/devops-python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/network) +[![LineCount](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2)](https://github.com/boyter/scc/) +[![Cocomo](https://sloc.xyz/github/HariSekhon/DevOps-Python-tools/?badge-bg-color=2081C2&category=cocomo)](https://github.com/boyter/scc/) +[![License](https://img.shields.io/github/license/HariSekhon/DevOps-Python-tools)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) +[![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/DevOps-Python-tools?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools/commits/master) + + + -A few of the Cloud, Big Data, NoSQL & Linux tools I've written over the years. All programs have `--help` to list the available options. +[![Codacy](https://app.codacy.com/project/badge/Grade/40a82d53f3394f4b99aa6eccb08e3c8d)](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard) +[![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools/badge)](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools) +[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_DevOps-Python-tools&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools) + +[![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Docker](https://img.shields.io/badge/container-Docker-blue?logo=docker&logoColor=white)](https://hub.docker.com/r/harisekhon/github/) +[![Dockerfile](https://img.shields.io/badge/repo-Dockerfiles-blue?logo=docker&logoColor=white)](https://github.com/HariSekhon/Dockerfiles) +[![DockerHub Pulls](https://img.shields.io/docker/pulls/harisekhon/pytools?label=DockerHub%20pulls&logo=docker&logoColor=white)](https://hub.docker.com/r/harisekhon/pytools) +[![DockerHub Build Automated](https://img.shields.io/docker/automated/harisekhon/pytools?logo=docker&logoColor=white)](https://hub.docker.com/r/harisekhon/pytools/) +[![StarTrack](https://img.shields.io/badge/Star-Track-blue?logo=github)](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,Dockerfiles&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,HAProxy-configs&r=HariSekhon,SQL-scripts) +[![StarCharts](https://img.shields.io/badge/Star-Charts-blue?logo=github)](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STARCHARTS.md) + -For many more tools see the [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) and [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) repos which contains many Hadoop, NoSQL, Web and infrastructure tools and Nagios plugins. +[![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://harisekhon.github.io/CI-CD/) +[![Jenkins](https://img.shields.io/badge/Jenkins-ready-blue?logo=jenkins&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile) +[![Concourse](https://img.shields.io/badge/Concourse-ready-blue?logo=concourse)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.concourse.yml) +[![GoCD](https://img.shields.io/badge/GoCD-ready-blue?logo=go)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.gocd.yml) +[![TeamCity](https://img.shields.io/badge/TeamCity-ready-blue?logo=teamcity)](https://github.com/HariSekhon/TeamCity-CI) + +[![CircleCI](https://circleci.com/gh/HariSekhon/DevOps-Python-tools.svg?style=svg)](https://circleci.com/gh/HariSekhon/DevOps-Python-tools) +[![BuildKite](https://img.shields.io/buildkite/8377537d0d9dddf4bf32826a6bf1c4e9ab88bc265007e1882c/master?label=BuildKite&logo=buildkite)](https://buildkite.com/hari-sekhon/devops-python-tools) +[![AppVeyor](https://img.shields.io/appveyor/build/harisekhon/DevOps-Python-tools/master?logo=appveyor&label=AppVeyor)](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master) +[![Drone](https://img.shields.io/drone/build/HariSekhon/DevOps-Python-tools/master?logo=drone&label=Drone)](https://cloud.drone.io/HariSekhon/DevOps-Python-tools) +[![Codefresh](https://g.codefresh.io/api/badges/pipeline/harisekhon/GitHub%2FDevOps-Python-tools?branch=master&key=eyJhbGciOiJIUzI1NiJ9.NWU1MmM5OGNiM2FiOWUzM2Y3ZDZmYjM3.O69674cW7vYom3v5JOGKXDbYgCVIJU9EWhXUMHl3zwA&type=cf-1)](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week) +[![Cirrus CI](https://img.shields.io/cirrus/github/HariSekhon/DevOps-Python-tools/master?logo=Cirrus%20CI&label=Cirrus%20CI)](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools) +[![Semaphore](https://harisekhon.semaphoreci.com/badges/DevOps-Python-tools.svg)](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools) +[![Buddy](https://img.shields.io/badge/Buddy-ready-1A86FD?logo=buddy)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml) +[![Shippable](https://img.shields.io/badge/Shippable-legacy-lightgrey?logo=jfrog&label=Shippable)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml) +[![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml) + +[![Azure DevOps Pipeline](https://dev.azure.com/harisekhon/GitHub/_apis/build/status/HariSekhon.DevOps-Python-tools?branchName=master)](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master) +[![GitLab Pipeline](https://img.shields.io/badge/GitLab%20CI-legacy-lightgrey?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines) +[![BitBucket Pipeline](https://img.shields.io/badge/Bitbucket%20CI-legacy-lightgrey?logo=bitbucket)](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/) +[![AWS CodeBuild](https://img.shields.io/badge/AWS%20CodeBuild-ready-blue?logo=amazon%20aws)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml) +[![GCP Cloud Build](https://img.shields.io/badge/GCP%20Cloud%20Build-ready-blue?logo=google%20cloud&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml) + +[![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/DevOps-Python-tools) +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools) +[![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/DevOps-Python-tools) + +[![ShellCheck](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml) +[![JSON](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml) +[![YAML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml) +[![XML](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml) +[![Markdown](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml) +[![Validation](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml) +[![Kics](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml) +[![Grype](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml) +[![Semgrep](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml) +[![Semgrep Cloud](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml) +[![Trivy](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml) + +[![Docker Build (Alpine)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml) +[![Docker Build (Debian)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml) +[![Docker Build (Fedora)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml) +[![Docker Build (Ubuntu)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml) + +[![GitHub Actions Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/GitHub%20Actions%20Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22) +[![Mac](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac.yaml) +[![Mac 11](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml) +[![Mac 12](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml) +[![Ubuntu](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22) +[![Ubuntu 20.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2020.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22) +[![Ubuntu 22.04](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Ubuntu%2022.04/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+22.04%22) +[![Debian](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22) +[![Debian 10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2010/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22) +[![Debian 11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2011/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22) +[![Debian 12](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Debian%2012/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+12%22) +[![Fedora](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Fedora/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Fedora%22) +[![Alpine](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine%22) +[![Alpine 3](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Alpine%203/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22) + +[![Python versions](https://img.shields.io/badge/Python-2.7+-3776AB?logo=python&logoColor=white)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Python 3.7](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.7/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22) +[![Python 3.8](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.8/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22) +[![Python 3.9](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.9/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22) +[![Python 3.10](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.10/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22) +[![Python 3.11](https://github.com/HariSekhon/DevOps-Python-tools/workflows/Python%203.11/badge.svg)](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22) + +[git.io/pytools](https://git.io/pytools) + +## AWS, Docker, Spark, Hadoop, HBase, Hive, Impala, Python & Linux Tools + +DevOps, Cloud, Big Data, NoSQL, Python & Linux tools. All programs have `--help`. Hari Sekhon Cloud & Big Data Contractor, United Kingdom -[https://www.linkedin.com/in/harisekhon](https://www.linkedin.com/in/harisekhon) -###### (you're welcome to connect with me on LinkedIn) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) +
*(you're welcome to connect with me on LinkedIn)* -##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. ##### +**Make sure you run `make update` if updating and not just `git pull` as you will often need the latest library submodule and possibly new upstream libraries** -### Quick Start ### +## Quick Start -#### Ready to run Docker image ##### +### Ready to run Docker image All programs and their pre-compiled dependencies can be found ready to run on [DockerHub](https://hub.docker.com/r/harisekhon/pytools/). List all programs: -``` + +```shell docker run harisekhon/pytools ``` + Run any given program: -``` + +```shell docker run harisekhon/pytools ``` -#### Automated Build from source ##### +### Automated Build from source +installs git, make, pulls the repo and build the dependencies: + +```shell +curl -L https://git.io/python-bootstrap | sh ``` -git clone https://github.com/harisekhon/devops-python-tools pytools + +or manually: + +```shell +git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools make ``` -Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/devops-python-tools#detailed-build-instructions) further down for more information. +To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension +instead of `.py`: + +```shell +make anonymize.pyc +``` + +Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/DevOps-Python-tools#detailed-build-instructions) further down for more information. -Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/harisekhon/devops-python-tools#jython-for-hadoop-utils) for details. +Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/HariSekhon/DevOps-Python-tools#jython-for-hadoop-utils) for details. -### Usage ### +### Usage -All programs come with a ```--help``` switch which includes a program description and the list of command line options. +All programs come with a `--help` switch which includes a program description and the list of command line options. -Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```. +Environment variables are supported for convenience and also to hide credentials from being exposed in the process list +eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and +often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`. -### DevOps Python Tools ### +### DevOps Python Tools - Inventory - Linux: - - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists) + - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing + - lists) - anonymizations include these and more: - hostnames / domains / FQDNs - email addresses @@ -70,101 +185,201 @@ Environment variables are supported for convenience and also to hide credentials - Kerberos principals - LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...) - Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings - - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to `````` - - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````) - - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications - - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters - - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files - - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename - - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) -- [AWS](https://aws.amazon.com/): - - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance) - - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys - - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove - - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days -- [Hadoop](http://hadoop.apache.org/) & NoSQL: - - [Spark](https://spark.apache.org/) & Data Format Converters: - - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter - - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter - - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas - - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas - - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter - - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter - - ```json_to_xml.py``` - JSON to XML converter - - ```xml_to_json.py``` - XML to JSON converter - - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' - - see also ```validate_*.py``` further down for all these formats and more + - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to `` + - placeholder tokens indicate what was stripped out (eg. ``, ``, ``) + - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes + communicating with each other to compare configs and log communications + - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support + teams can differentiate hosts in clusters + - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel + before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation + important for anonymization rules, as well as maintaining file content order. On servers this parallelization can + result in a 30x speed up for large log files + - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file + basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename + - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments, + useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). + Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See + further down for more details and sub-programs that simplify usage for many of the most common cluster technologies + - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in + your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo) +- [Amazon Web Services](https://aws.amazon.com/): + - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally + filters for active keys and older than N days (for key rotation governance) + - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have + never been used (these should generally be removed/disabled). Optionally filters for only active keys + - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys. + Optionally filters for users not used in the last N days to find old accounts to remove + - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for + users with passwords not used in the last N days +- [Google Cloud Platform](https://cloud.google.com/): + - [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python: + - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to + [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is + triggered by [Cloud Scheduler](https://cloud.google.com/scheduler) + - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and + [Cloud Scheduler](https://cloud.google.com/scheduler) jobs + - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP + address - use this to test your VPC connector public routing, comparison with firewall rules etc. + - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its + accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via + VPC connector routing + - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with + their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days +- [Docker](https://www.docker.com/): + - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a + docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very + useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only + the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests + across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh` + - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official + `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned + to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy + shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. + `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all + docker images eg. `docker_pull_all_images_all_tags.sh` + - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION` +- [Spark](https://spark.apache.org/) & Data Format Converters: + - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter + - `spark_parquet_to_avro.py` - PySpark Parquet => Avro converter + - `spark_csv_to_avro.py` - PySpark CSV => Avro converter, supports both inferred and explicit schemas + - `spark_csv_to_parquet.py` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas + - `spark_json_to_avro.py` - PySpark JSON => Avro converter + - `spark_json_to_parquet.py` - PySpark JSON => Parquet converter + - `xml_to_json.py` - XML to JSON converter + - `json_to_xml.py` - JSON to XML converter + - `json_to_yaml.py` - JSON to YAML converter + - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for + pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and + [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / + directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard + output for convenient command line chaining and redirection, optionally continues on error, collects broken records + to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not + technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json' + - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON) + - see also `validate_*.py` further down for all these formats and more +- [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL: - [Ambari](https://hortonworks.com/apache/ambari/): - - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API + - `ambari_blueprints.py` - Blueprint cluster templating and deployment tool using Ambari API - list blueprints - fetch all blueprints or a specific blueprint to local json files - blueprint an existing cluster - create a new cluster using a blueprint - - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control + - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for + proper revision control - optionally strips out the excessive and overly specific configs to create generic more reusable templates - - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool - - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts - - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API - - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API + - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable + using this tool + - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts + - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API + - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API - [Hadoop](http://hadoop.apache.org/) HDFS: - - ```hdfs_find_replication_factor_1.py``` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows - - ```hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. - - ```hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum) - - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files + - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to + replication factor 3 to avoid missing block alerts during datanode maintenance windows + - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file + or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data + layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports. + - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster + than doing `hdfs dfs -cat | md5sum`) + - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree + showing block size, replication factor, underfilled blocks and small files - [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/): - - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output + - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format + to standard output The following programs can all optionally filter by database / table name regex: - - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table - - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations - - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs - - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs - - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala + table + - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation + between cluster migrations + - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for + finding unusually wide tables + - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts. + Useful for finding unusually big tables + - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where + any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or + subtle ETL bugs + - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs. + Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain + NULLs. Useful for catching data quality problems or subtle ETL bugs + - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL + property from describe table + - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location - [HBase](https://hbase.apache.org/): - - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. - - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables - - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions - - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix - - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables. - - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables. - - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending - - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting - - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting - - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold) + - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, + with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI + tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written. + - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting + tables + - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table, + giving per region row counts and % of total rows for the table as well as median and quartile row counts per + regions + - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in + an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row + counts per prefix + - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating + on all tables or takes an optional regex and compacts only matching tables. + - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an + optional regex and flushes only matching tables. + - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or + memStoreSize, ascending or descending + - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average + since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / + total requests per second. Useful for watching more granular region stats to detect region hotspotting + - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or + average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, + write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular + RegionServer stats to detect RegionServer hotspotting + - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received + the least requests (requests below a given threshold) - [OpenTSDB](http://opentsdb.net/): - - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting - - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending + - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew + and help avoid HBase region hotspotting + - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase + tables with optionally their created date, sorted ascending - [Pig](https://pig.apache.org/) - - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch) - - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) - - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop - - ```ipython-notebook-pyspark.py``` - per-user authenticated IPython Notebook + PySpark integration to allow each user to auto-create their own password protected IPython Notebook running Spark - - ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch) - - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required - - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA - - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA - - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA - - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing) - - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing) - - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node - - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node - - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore - - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator - - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server - - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server - - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node - - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node - - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) -- [Docker](https://www.docker.com/): - - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh``` - - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (official `docker search` is limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh``` - - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION``` + - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Elasticsearch](https://www.elastic.co/products/elasticsearch) + - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to + [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud) + - `pig_udfs.jy` - Pig Jython UDFs for Hadoop +- `find_active_server.py` - returns first available healthy server or active master in high availability deployments, + useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex + content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for + which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you + want to query cluster wide information available from any online peer (eg. Elasticsearch) + - The following are simplified specialisations of the above program, just pass host arguments, all the details have + been baked in, no switches required + - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA + - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA + - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA + - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run + multiple of these for load balancing) + - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server + (run multiple of these for load balancing) + - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node + - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node + - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, + Catalog or Statestore + - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator + - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server + - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server + - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node + - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node + - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the + [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins) - [Travis CI](https://travis-ci.org/): - - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red - - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI + - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - + useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red + - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks + session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot + debug launcher for Travis CI +- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as + Chrome and Firefox to fetch a given URL and content/regex match the result - Data Validation (useful in CI): - - ```validate_*.py``` - validate files, directory trees and/or standard input streams + - `validate_*.py` - validate files, directory trees and/or standard input streams - supports the following file formats: - Avro - CSV @@ -174,40 +389,44 @@ Environment variables are supported for convenience and also to hide credentials - Parquet - XML - YAML - - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) - - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies + - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, + `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`) + - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for + things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images + [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & + Linux technologies ### Detailed Build Instructions -##### Python VirtualEnv localized installs - -The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages instead of installing from PyPI then follow the Manual Build section below. +#### Python VirtualEnv localized installs +The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it +detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages +instead of installing from PyPI then follow the Manual Build section below. -#### Manual Setup +### Manual Setup Enter the pytools directory and run git submodule init and git submodule update to fetch my library repo: -``` -git clone https://github.com/harisekhon/devops-python-tools pytools +```shell +git clone https://github.com/HariSekhon/DevOps-Python-tools pytools cd pytools git submodule init git submodule update sudo pip install -r requirements.txt ``` - -#### Offline Setup +### Offline Setup Download the DevOps Python Tools and Pylib git repos as zip files: -https://github.com/HariSekhon/devops-python-tools/archive/master.zip + -https://github.com/HariSekhon/pylib/archive/master.zip + -Unzip both and move Pylib to the ```pylib``` folder under DevOps Python Tools. +Unzip both and move Pylib to the `pylib` folder under DevOps Python Tools. -``` +```shell unzip devops-python-tools-master.zip unzip pylib-master.zip @@ -216,156 +435,246 @@ mv -v pylib-master pylib mv -vf pylib pytools/ ``` -Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). +Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal +mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions). All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files. Internal Mirror example ([JFrog Artifactory](https://jfrog.com/artifactory/) or similar): -``` +```shell sudo pip install --index https://host.domain.com/api/pypi/repo/simple --trusted host.domain.com -r requirements.txt ``` Proxy example: -``` +```shell sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt ``` -##### Mac OS X +#### Mac OS X -The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing `git` is enough to trigger Xcode install). +The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing +`git` is enough to trigger Xcode install). -I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): +I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for +development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X): -``` -bash-tools/setup/install_homebrew.sh +```shell +bash-tools/install/install_homebrew.sh ``` -``` +```shell brew install openssl wget ``` If failing to build an OpenSSL lib dependency, just prefix the build command like so: -``` +```shell sudo OPENSSL_INCLUDE=/usr/local/opt/openssl/include OPENSSL_LIB=/usr/local/opt/openssl/lib ... ``` -You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is caused by pip 10 vs pip 9 and downgrading will work around it: +You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is +caused by pip 10 vs pip 9 and downgrading will work around it: -``` +```shell sudo pip install --upgrade pip==9.0.1 make sudo pip install --upgrade pip make ``` -### Jython for Hadoop Utils ### +### Jython for Hadoop Utils The 3 Hadoop utility programs listed below require Jython (as well as Hadoop to be installed and correctly configured) -``` +```shell hdfs_time_block_reads.jy hdfs_files_native_checksums.jy hdfs_files_stats.jy ``` Run like so: -``` + +```shell jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help ``` -The ```-J-cp $(hadoop classpath)``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs. +The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop +APIs. See below for procedure to install Jython if you don't already have it. -##### Automated Jython Install +#### Automated Jython Install This will download and install jython to /opt/jython-2.7.0: -``` +```shell make jython ``` -##### Manual Jython Install +#### Manual Jython Install -Jython is a simple download and unpack and can be fetched from http://www.jython.org/downloads.html +Jython is a simple download and unpack and can be fetched from Then add the Jython install bin directory to the $PATH or specify the full path to the `jython` binary, eg: -``` +```shell /opt/jython-2.7.0/bin/jython hdfs_time_block_reads.jy ... ``` +### Configuration for Strict Domain / FQDN validation -#### Configuration for Strict Domain / FQDN validation #### - -Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/harisekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments). +Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my +[PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, +`.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal +environments). -#### Python SSL certificate verification problems +### Python SSL certificate verification problems If you end up with an error like: -``` + +```shell ./dockerhub_show_tags.py centos ubuntu [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:765) ``` -It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One quick fix is to do the following: -``` + +It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One +quick fix is to do the following: + +```shell sudo pip uninstall -y certifi && sudo pip install certifi==2015.04.28 ``` -### Updating ### +### Updating -Run ```make update```. This will git pull and then git submodule update which is necessary to pick up corresponding library updates. +Run: -If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run ```make update-no-recompile``` (will miss new library dependencies - do full ```make update``` if you encounter issues). +```shell +make update +``` + +This will git pull and then git submodule update which is necessary to pick up corresponding library updates. + +If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each +time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter +issues). ### Testing [Continuous Integration](https://travis-ci.org/HariSekhon/devops-python-tools) is run on this repo with tests for success and failure scenarios: -- unit tests for the custom supporting [python library](https://github.com/harisekhon/pylib) + +- unit tests for the custom supporting [python library](https://github.com/HariSekhon/pylib) - integration tests of the top level programs using the libraries for things like option parsing -- [functional tests](https://github.com/HariSekhon/devops-python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/) +- [functional tests](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/) To trigger all tests run: -``` +```shell make test ``` -which will start with the underlying libraries, then move on to top level integration tests and functional tests using docker containers if docker is available. +which will start with the underlying libraries, then move on to top level integration tests and functional tests using +docker containers if docker is available. -### Contributions ### +### Contributions Patches, improvements and even general feedback are welcome in the form of GitHub pull requests and issue tickets. -### See Also ### +You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another +Hortonworks guy Jonas Straub: + + + +## Star History + +[![Star History Chart](https://api.star-history.com/svg?repos=HariSekhon/DevOps-Python-tools&type=Date)](https://star-history.com/#HariSekhon/DevOps-Python-tools&Date) + +[git.io/python-tools](https://git.io/python-tools) + +[git.io/pytools](https://git.io/pytools) + +## More Core Repos + + + +### Knowledge +[![Knowledge-Base](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Diagrams-as-Code](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) -* [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) + -* [PyLib](https://github.com/harisekhon/pylib) - Python library leveraged throughout the programs in this repo as a submodule +### DevOps Code -* [Perl Lib](https://github.com/harisekhon/lib) - Perl version of above library +[![DevOps-Bash-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) +[![DevOps-Python-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Python-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Python-tools) +[![DevOps-Perl-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Perl-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Perl-tools) +[![DevOps-Golang-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Golang-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Golang-tools) -You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub: +### Containerization + +[![Kubernetes-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Dockerfiles](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) + +### CI/CD + +[![GitHub-Actions](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Jenkins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) + +### Databases - DBA - SQL + +[![SQL-scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) + +### DevOps Reloaded + +[![HAProxy-configs](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Terraform](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Packer](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Ansible](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Environments](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) + +### Monitoring + +[![Nagios-Plugins](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Nagios-Plugin-Kafka](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Prometheus](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) + +### Templates + +[![Templates](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Template-repo](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) + +### Desktop + +[![TamperMonkey](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Hammerspoon](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) +[![MPV-Scripts](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=MPV-Scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/MPV-Scripts) + +### Spotify + +[![Spotify-tools](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Spotify-playlists](https://github-readme-stats-fast.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) -* https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb +The rest of my original source repos are +[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). -### Stargazers over time +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/) +and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo. -[![Stargazers over time](https://starchart.cc/HariSekhon/DevOps-Python-tools.svg)](https://starchart.cc/HariSekhon/DevOps-Python-tools) + diff --git a/ambari_ams_metrics.sh b/ambari_ams_metrics.sh index 85f24ee53..64f20f0a2 100755 --- a/ambari_ams_metrics.sh +++ b/ambari_ams_metrics.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2018-07-16 22:14:34 +0100 (Mon, 16 Jul 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # There were 2346 metrics last I checked but this probably varies a lot based on what services are deployed diff --git a/ambari_blueprints.py b/ambari_blueprints.py index 643bc3374..e5dd52217 100755 --- a/ambari_blueprints.py +++ b/ambari_blueprints.py @@ -1,18 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-11-08 14:09:50 +0000 (Sun, 08 Nov 2015) # (re-instantiated from a Perl version in 2014) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -90,7 +90,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.2' +__version__ = '0.10.3' class AmbariBlueprintTool(CLI): @@ -116,7 +116,7 @@ def connection(self, host, port, user, password, ssl=False, **kwargs): #log.info("contacting Ambari as '%s'" % self.user) if not isHost(host) or not isPort(port) or not isUser(user) or not password: raise InvalidOptionException('invalid options passed to AmbariBlueprint()') - proto = 'http' # pylint: disable=unused-variable + proto = 'http' # pylint: disable=unused-variable,possibly-unused-variable if ssl: proto = 'https' self.host = host @@ -140,8 +140,8 @@ def connection(self, host, port, user, password, ssl=False, **kwargs): except IOError as _: die("'failed to create dir '%s': %s" % (self.blueprint_dir, _)) - # TODO: change to @staticmethod - def parse_cluster_name(self, item): # pylint: disable=no-self-use + @staticmethod + def parse_cluster_name(item): if isStr(item): item = json.loads(item) try: @@ -154,8 +154,8 @@ def get_clusters(self): json_data = self.list('clusters') return [self.parse_cluster_name(item) for item in json_data['items']] - # TODO: change to @staticmethod - def parse_blueprint_name(self, item): # pylint: disable=no-self-use + @staticmethod + def parse_blueprint_name(item): if isStr(item): item = json.loads(item) try: @@ -168,8 +168,8 @@ def get_blueprints(self): json_data = self.list('blueprints') return [self.parse_blueprint_name(item) for item in json_data['items']] - # TODO: change to @staticmethod - def parse_host_name(self, item): # pylint: disable=no-self-use + @staticmethod + def parse_host_name(item): if isStr(item): item = json.loads(item) try: @@ -320,7 +320,7 @@ def send_blueprint_file(self, filename, name=''): except KeyError as _: pass if not name: - name = os.path.splitext(os.path.basename(file))[0] + name = os.path.splitext(os.path.basename(filename))[0] log.info("name not specified and couldn't determine blueprint name from blueprint data, reverting to using filename without extension '%s'" % name) # pylint: disable=line-too-long # this solves the issue of having duplicate Blueprint.blueprint_name keys try: @@ -329,7 +329,7 @@ def send_blueprint_file(self, filename, name=''): data = json.dumps(json_data) log.info("reset blueprint field name to '%s'" % name) except ValueError as _: - qquit('CRITICAL', "invalid json found in file '%s': %s" % (file, name)) + qquit('CRITICAL', "invalid json found in file '%s': %s" % (filename, name)) except KeyError as _: log.warn('failed to reset the Blueprint name: %s' % _) return self.send_blueprint(name, data) @@ -399,8 +399,8 @@ def save_cluster(self, cluster, path=''): log.debug("cluster '%s' blueprint content = '%s'" % (cluster, data)) self.save(cluster, path, data) - # TODO: change to @staticmethod - def save(self, name, path, data): # pylint: disable=no-self-use + @staticmethod + def save(name, path, data): # log.debug('save(%s, %s)' % (name, data)) if data is None: err = "blueprint '%s' returned None" % name diff --git a/ambari_cancel_all_requests.sh b/ambari_cancel_all_requests.sh index 15e079525..7d2df4425 100755 --- a/ambari_cancel_all_requests.sh +++ b/ambari_cancel_all_requests.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-09-27 17:25:36 +0100 (Tue, 27 Sep 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/ambari_trigger_service_checks.py b/ambari_trigger_service_checks.py index edf98b2bf..2afe64207 100755 --- a/ambari_trigger_service_checks.py +++ b/ambari_trigger_service_checks.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-09-23 15:45:28 +0200 (Fri, 23 Sep 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/anonymize.py b/anonymize.py index 94beaf3f6..ebcb20ab0 100755 --- a/anonymize.py +++ b/anonymize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # @@ -6,16 +6,16 @@ # Date: 2018-08-08 19:02:02 +0100 (Wed, 08 Aug 2018) # Original Date: 2013-07-18 21:17:41 +0100 (Thu, 18 Jul 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # -# ported from Perl version from DevOps Perl Tools repo (https://github.com/harisekhon/devops-perl-tools) +# ported from Perl version from DevOps Perl Tools repo (https://github.com/HariSekhon/DevOps-Perl-tools) # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # @@ -36,7 +36,7 @@ Ignore phrases are in a similar file anonymize_ignore.conf, also adjacent to this program. -Based on Perl Anonymize.pl from https://github.com/harisekhon/devops-perl-tools +Based on Perl Anonymize.pl from https://github.com/HariSekhon/DevOps-Perl-tools The Perl version is incredibly faster than Python due to the better regex engine @@ -72,25 +72,30 @@ # pylint: disable=unused-import from harisekhon.utils import \ aws_host_ip_regex, \ - domain_regex, \ domain_regex_strict, \ - email_regex, \ filename_regex, \ fqdn_regex, \ host_regex, \ hostname_regex, \ ip_prefix_regex, \ ip_regex, \ - mac_regex, \ subnet_mask_regex, \ user_regex + # used dynamically + # pylint: disable=unused-import + # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend + from harisekhon.utils import \ + domain_regex, \ + email_regex, \ + mac_regex \ + # lgtm [py/unused-import] - used by dynamic code so code analyzer cannot comprehend from harisekhon import CLI except ImportError as _: print(traceback.format_exc(), end='') sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.10.10' +__version__ = '0.11.0' ip_regex = r'(?!127\.0\.0\.)' + ip_regex subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex @@ -219,7 +224,7 @@ def __init__(self): #'countryCode', 'displayName', 'displayNamePrintable', - 'division' + 'division', 'employeeID', 'groupMembershipSAM', 'info', @@ -353,8 +358,8 @@ def __init__(self): 'group2': r'({group_name}{sep}){user}'.format(group_name=group_name, sep=arg_sep, user=user_regex), 'group3': r'for\s+group\s+{group}'.format(group=user_regex), 'group4': r'(["\']{group_name}["\']\s*:\s*["\']?){group}'.format(group_name=group_name, group=user_regex), - 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format(\ - group='({}/)*{}'.format(user_regex, user_regex)), + 'group5': r'(arn:aws:iam:[^:]*:)\d+(:group/){group}'.format( + group='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)), 'user': r'([-\.]{user_name}{sep})\S+'.format(user_name=user_name, sep=arg_sep), 'user2': r'/(home|user)/{user}'.format(user=user_regex), 'user3': r'({user_name}{sep}){user}'.format(user_name=user_name, sep=arg_sep, user=user_regex), @@ -366,7 +371,8 @@ def __init__(self): 'user7': r'(["\'](?:{user_name}|owner)["\']\s*:\s*["\']?){user}'\ .format(user_name=user_name, user=user_regex), #'user8': r'arn:aws:iam::\d{12}:user/{user}'.format(user=user_regex), - 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format(user='({}/)*{}'.format(user_regex, user_regex)), + 'user8': r'(arn:aws:iam:[^:]*:)\d+(:user/){user}'.format( + user='({user_regex}/)*{user_regex}'.format(user_regex=user_regex)), 'password': r'([\.-]?{pass_word_phrase}{sep}){pw}'\ .format(pass_word_phrase=pass_word_phrase, sep=arg_sep, @@ -377,6 +383,9 @@ def __init__(self): sep=arg_sep, pass_word_phrase=pass_word_phrase, pw=password_quoted), + 'password4': r'([\.-]?(?:api-?)?token{sep}){pw}'\ + .format(sep=arg_sep, + pw=password_quoted), 'ip': r'(?', 'password2': r'\1:', 'password3': r'\1\2', + 'password4': r'\1', 'ip': r'/', 'ip2': r'', 'ip3': r'', @@ -650,6 +660,7 @@ def process_options(self): self.anonymizations['ip'] = False else: for _ in self.anonymizations: + # pylint: disable=no-else-continue if _ in ('subnet_mask', 'mac', 'group'): continue elif _ == 'database': @@ -781,46 +792,47 @@ def run(self): # allow to easily switch pre-compilation on/off for testing # testing shows on a moderate sized file that it is a couple secs quicker to use pre-compiled regex def compile(self, name, regex): + log.debug(f"compiling regex '{name}' = '{regex}'") self.regex[name] = re.compile(regex, re.I) #self.regex[name] = regex def prepare_regex(self): self.compile('hostname', - r'(? 2018-01-:00:00 - r'(?!\d+T\d+:\d+)' + \ - r'(?!\d+[^A-Za-z0-9]|' + \ - self.custom_ignores_raw + ')' + \ - '(' + hostname_regex + ')' + \ + r'(?!\d+T\d+:\d+)' + + r'(?!\d+[^A-Za-z0-9]|' + + self.custom_ignores_raw + ')' + + '(' + hostname_regex + ')' + self.negative_host_lookbehind + r':(\d{1,5}(?!\.?\w))', ) self.compile('domain', # don't match java -some.net.property - #r'(?/dev/null || : echo - "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename" + "$srcdir/bash-tools/bin/split.sh" --parts "$parallelism" "$filename" echo "Anonymizing parts" for file_part in "$filename".*; do cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized" diff --git a/aws_s3_presign.py b/aws_s3_presign.py index 2fbb1331f..e1addfbbc 100755 --- a/aws_s3_presign.py +++ b/aws_s3_presign.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-01-14 17:45:38 +0000 (Tue, 14 Jan 2020) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py index c0026570d..be218c2db 100755 --- a/aws_users_access_key_age.py +++ b/aws_users_access_key_age.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-12-13 17:24:40 +0000 (Fri, 13 Dec 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -35,9 +35,9 @@ See also: aws_users_access_key_age.sh - simpler version in the adjacent DevOps Bash Tools repo without age filtering - - https://github.com/harisekhon/devops-bash-tools + - https://github.com/HariSekhon/DevOps-Bash-tools -Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins) +Advanced Nagios Plugins (https://github.com/HariSekhon/Nagios-Plugins) check_aws_access_keys_age.py check_aws_access_keys_disabled.py diff --git a/aws_users_last_used.py b/aws_users_last_used.py index 2657f03f7..e06a7bcdb 100755 --- a/aws_users_last_used.py +++ b/aws_users_last_used.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019) # -# https://github.com/harisekhon/nagios-plugins +# https://github.com/HariSekhon/Nagios-Plugins # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -35,8 +35,8 @@ See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools -- https://github.com/harisekhon/devops-bash-tools -- https://github.com/harisekhon/nagios-plugins +- https://github.com/HariSekhon/DevOps-Bash-tools +- https://github.com/HariSekhon/Nagios-Plugins """ @@ -54,7 +54,6 @@ from io import StringIO from math import floor import boto3 -from botocore.exceptions import ClientError srcdir = os.path.abspath(os.path.dirname(__file__)) libdir = os.path.join(srcdir, 'pylib') sys.path.append(libdir) @@ -67,7 +66,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.1.0' +__version__ = '0.1.1' class AWSUsersLastUsed(CLI): @@ -103,16 +102,16 @@ def run(self): break log.info('waiting for credentials report') time.sleep(1) - try: - result = iam.get_credential_report() - except ClientError as _: - raise + #try: + result = iam.get_credential_report() + #except ClientError as _: + # raise csv_content = result['Content'] log.debug('%s', csv_content) filehandle = StringIO(unicode(csv_content)) filehandle.seek(0) csvreader = csv.reader(filehandle) - headers = csvreader.next() + headers = next(csvreader) assert headers[0] == 'user' assert headers[4] == 'password_last_used' assert headers[10] == 'access_key_1_last_used_date' diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py index 7e00ee4e0..6810abd17 100755 --- a/aws_users_pw_last_used.py +++ b/aws_users_pw_last_used.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-12-19 11:43:25 +0000 (Thu, 19 Dec 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -31,9 +31,9 @@ See also the DevOps Bash Tools repo and The Advanced Nagios Plugins Collection for similar tools -https://github.com/harisekhon/devops-bash-tools +https://github.com/HariSekhon/DevOps-Bash-tools -https://github.com/harisekhon/nagios-plugins +https://github.com/HariSekhon/Nagios-Plugins """ diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py index 9b9083eb9..befb152ee 100755 --- a/aws_users_unused_access_keys.py +++ b/aws_users_unused_access_keys.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019) # -# https://github.com/harisekhon/nagios-plugins +# https://github.com/HariSekhon/Nagios-Plugins # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -33,8 +33,8 @@ See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools -- https://github.com/harisekhon/devops-bash-tools -- https://github.com/harisekhon/nagios-plugins +- https://github.com/HariSekhon/DevOps-Bash-tools +- https://github.com/HariSekhon/Nagios-Plugins """ @@ -65,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.0' +__version__ = '0.2.1' class AWSUnusedAccessKeys(CLI): @@ -113,7 +113,7 @@ def run(self): filehandle = StringIO(unicode(csv_content)) filehandle.seek(0) csvreader = csv.reader(filehandle) - headers = csvreader.next() + headers = next(csvreader) log.debug('headers: %s', headers) assert headers[0] == 'user' assert headers[8] == 'access_key_1_active' diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..b430c58dd --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,70 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: Sun Feb 23 19:02:10 2020 +0000 +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# A z u r e D e v O p s P i p e l i n e +# ============================================================================ # + +# https://aka.ms/yaml + +trigger: + - master + +variables: + # ubuntu version + os_version: '22.04' + +pool: + # there is no /dev/stderr on this azure build! + #vmImage: 'ubuntu-latest' + #vmImage: 'ubuntu-22.04' + vmImage: 'ubuntu-$(os_version)' + +# unprivileged container without sudo, cannot install dependencies +#container: ubuntu:22.04 + +steps: + - script: cat /etc/*-release + displayName: OS Release + + # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName' + - script: env | sort + displayName: Environment + + # doesn't work in container due to unprivileged execution and lack of sudo + #- script: sudo apt-get update && sudo apt-get install -y git make + # displayName: install git & make + + #- script: make + # displayName: build + + # doesn't work in vmImage build due to lack of access to normal /dev/stderr device + # tee: /dev/stderr: No such device or address + #- script: make test + # displayName: test + + # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-( + - script: | + sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c ' + set -ex + cd /code + setup/ci_bootstrap.sh + if [ -x setup/ci_git_set_dir_safe.sh ]; then + setup/ci_git_set_dir_safe.sh + fi + make init + make ci test + ' + displayName: docker build diff --git a/bash-tools b/bash-tools index 1e33eaec4..11dee29ce 160000 --- a/bash-tools +++ b/bash-tools @@ -1 +1 @@ -Subproject commit 1e33eaec4117818ee32e3644e101a715f5a66de0 +Subproject commit 11dee29cea607445270d8bd675d1bcdecb069c14 diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml new file mode 100644 index 000000000..cdf5243ee --- /dev/null +++ b/bitbucket-pipelines.yml @@ -0,0 +1,38 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 17:08:57 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# B i t b u c k e t C I / C D P i p e l i n e +# ============================================================================ # + +# Reference: +# +# https://support.atlassian.com/bitbucket-cloud/docs/configure-bitbucket-pipelinesyml/ + +# Languages: +# +# https://confluence.atlassian.com/x/5Q4SMw + +# You can specify a custom docker image from Docker Hub as your build environment. +image: atlassian/default-image:2 + +pipelines: + default: + - step: + script: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test diff --git a/boot b/boot new file mode 120000 index 000000000..4092e5539 --- /dev/null +++ b/boot @@ -0,0 +1 @@ +setup/bootstrap.sh \ No newline at end of file diff --git a/buddy.yml b/buddy.yml new file mode 100644 index 000000000..e0b57451f --- /dev/null +++ b/buddy.yml @@ -0,0 +1,48 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-16 14:02:53 +0000 (Mon, 16 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# B u d d y C I +# ============================================================================ # + +# https://buddy.works/docs/yaml/yaml-schema + +--- +- pipeline: "Build" + trigger_mode: "ON_EVERY_PUSH" + ref_name: "master" + ref_type: "BRANCH" + target_site_url: "https://github.com/HariSekhon/DevOps-Python-tools" + trigger_condition: "ALWAYS" + actions: + - action: "Execute: make ci test" + type: "BUILD" + working_directory: "/buddy/DevOps-Python-tools" + docker_image_name: "library/ubuntu" + docker_image_tag: "18.04" + #setup_commands: + # this step gets cached, which results in + # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? + # - apt update + # - apt install -qy git make + execute_commands: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test + volume_mappings: + - "/:/buddy/DevOps-Python-tools" + shell: "BASH" + trigger_condition: "ALWAYS" diff --git a/center.py b/center.py index 752db7f6a..379353e68 100755 --- a/center.py +++ b/center.py @@ -1,17 +1,19 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et +# args: -s <<< "Auth & Config" +# args: -s <<< "GKE Clusters" # # Author: Hari Sekhon # Date: 2016-01-29 21:05:38 +0000 (Fri, 29 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -42,7 +44,8 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.4.0' +__version__ = '0.5.1' + class Center(CLI): @@ -51,8 +54,13 @@ def __init__(self): super(Center, self).__init__() # Python 3.x # super().__init__() - self.re_bound = re.compile(r'(\b)') + # this doesn't put enough spaces around ampersands, eg. in "Auth & Config" + #self.re_bound = re.compile(r'(\b)') + self.re_spaces = re.compile(r'(\s)') + self.re_multiple_spaces = re.compile(r'(\s){2}') self.re_chars = re.compile(r'([^\s])(?!\s)') + self.re_chars_spaced = re.compile(r'([^\s])\s') + self.timeout_default = None def add_options(self): self.add_opt('-w', '--width', default=80, type='int', metavar='', @@ -61,11 +69,16 @@ def add_options(self): help='No comment prefix handling') self.add_opt('-s', '--space', action='store_true', default=False, help='Space all chars out, makes bigger headings') + self.add_opt('-u', '--unspace', action='store_true', default=False, + help='Removes spaces betweeen chars out, the inverse of --space') def run(self): log_option('width', self.get_opt('width')) log_option('no comment prefix', self.get_opt('no_comment')) log_option('space chars', self.get_opt('space')) + log_option('unspace chars', self.get_opt('unspace')) + if self.get_opt('space') and self.get_opt('unspace'): + self.usage("--space and --unspace are mutually exclusive!") if self.args: self.process_line(' '.join(self.args)) else: @@ -73,10 +86,16 @@ def run(self): self.process_line(line) def space(self, line): - line = self.re_bound.sub(r' ', line) + #line = self.re_bound.sub(r' ', line) + line = self.re_spaces.sub(r'\1\1\1', line) line = self.re_chars.sub(r'\1 ', line) return line + def unspace(self, line): + line = self.re_chars_spaced.sub(r'\1', line) + line = self.re_multiple_spaces.sub(r'\1', line) + return line + def process_line(self, line): char = '' if not line: @@ -85,7 +104,7 @@ def process_line(self, line): char = ' ' # preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it line = line.strip() - if isChars(line[0], '#'): + if line and isChars(line[0], '#'): char = line[0] line = line.lstrip(char) elif len(line) > 1 and isChars(line[0:1], '/'): @@ -96,9 +115,12 @@ def process_line(self, line): line = line.lstrip(char) if self.get_opt('space'): line = self.space(line) + if self.get_opt('unspace'): + line = self.unspace(line) line = line.strip() side = int(max((self.get_opt('width') - len(line)) / 2, 0)) print(char + ' ' * side + line) + if __name__ == '__main__': Center().main() diff --git a/cicd/.concourse.yml b/cicd/.concourse.yml new file mode 100644 index 000000000..92847121e --- /dev/null +++ b/cicd/.concourse.yml @@ -0,0 +1,65 @@ +# +# Author: Hari Sekhon +# Date: 2020-03-21 11:06:48 +0000 (Sat, 21 Mar 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# C o n c o u r s e C I +# ============================================================================ # + +# https://concourse-ci.org/golang-library-example.html + +# https://resource-types.concourse-ci.org/ +# https://concourse-ci.org/resource-types.html +resources: + - name: github + icon: github-circle + type: git + source: + uri: https://github.com/HariSekhon/DevOps-Python-tools + branch: master + #- name: daily + # type: time + # source: + # interval: 1d + +# https://concourse-ci.org/jobs.html +jobs: + - name: build + public: false + plan: + - get: github # from resource above + trigger: true + #version: every # build every git commit, default: latest + - task: build + config: + platform: linux + image_resource: + type: docker-image + source: + repository: ubuntu + tag: latest + inputs: + - name: github + path: code + params: + CONCOURSE: 1 + run: + path: /bin/bash + args: + - -c + - | + cd code && + setup/ci_bootstrap.sh && + make init && + make ci test diff --git a/cicd/.gocd.yml b/cicd/.gocd.yml new file mode 100644 index 000000000..65583a818 --- /dev/null +++ b/cicd/.gocd.yml @@ -0,0 +1,94 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G o C D +# ============================================================================ # + +# https://github.com/tomzo/gocd-yaml-config-plugin#setup + +# https://docs.gocd.org/current/configuration/configuration_reference.html + +--- +format_version: 3 +pipelines: + DevOps-Python-tools: + group: defaultGroup + label_template: ${COUNT} + lock_behavior: none + display_order: -1 + materials: + git: + git: https://github.com/HariSekhon/DevOps-Python-tools + shallow_clone: false + auto_update: true + branch: master + stages: + - build-and-test: + fetch_materials: true + keep_artifacts: false + clean_workspace: false + approval: + type: success + allow_only_on_success: false + jobs: + #apt-update: + # timeout: 10 + # tasks: + # - exec: + # command: apt + # arguments: + # - update + # run_if: passed + #install-make: + # timeout: 10 + # tasks: + # - exec: + # command: apt + # arguments: + # - install + # - -qy + # - git + # - make + # run_if: passed + ci-bootstrap: + timeout: 10 + tasks: + - exec: + command: setup/ci_bootstrap.sh + run_if: passed + init: + timeout: 10 + tasks: + - exec: + command: make + arguments: + - init + run_if: passed + build: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - ci + run_if: passed + test: + timeout: 60 + tasks: + - exec: + command: make + arguments: + - test + run_if: passed diff --git a/cicd/buildspec.yml b/cicd/buildspec.yml new file mode 100644 index 000000000..f4e887953 --- /dev/null +++ b/cicd/buildspec.yml @@ -0,0 +1,66 @@ +# +# Author: Hari Sekhon +# Date: 2020-12-19 15:32:28 +0000 (Sat, 19 Dec 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# A W S C o d e B u i l d +# ============================================================================ # + +# References: +# +# https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html + +# Operating System should be set to Ubuntu, not Amazon Linux 2 +# - this is both recommended since programming language runtimes are now included in standard image of ubuntu, but also to avoid this error: +# +# /usr/bin/amazon-linux-extras +# /root/.pyenv/versions/3.8.3/bin/python: No module named amazon_linux_extras + +version: 0.2 + +# only on Linux, the user to run as - global setting, alternatively set inside a phase section below for localized user +#run-as: linux-username + +env: + shell: bash + # don't store sensitive stuff like AWS secret keys in variables, use parameter-store or secrets-manager + # any environment variables replace existing environment variables, ie. beware if setting PATH that it'll replace the existing PATH with a non-interpolated literal + # project env vars take precedence over these, with start build vars taking highest precedence + #variables: + # DEBUG: "1" + #exported-variables: + # - DEBUG + +phases: + # install prerequisites / languages / frameworks / packages to allow build to work + install: + #commands: + # - setup/ci_bootstrap.sh + # languages to install + runtime-versions: + #java: openjdk11 + # AWS LTS release of OpenJDK 11 + java: corretto11 + golang: 1.14 + python: 3.8 + ruby: 2.7 + # eg. sign in to Amazon ECR or install package dependencies + pre_build: + commands: + - setup/ci_bootstrap.sh + build: + commands: + - echo Build started on `date` + - make + - echo Build completed on `date` + - make test + - echo Tests completed on `date` diff --git a/cicd/cloudbuild.yaml b/cicd/cloudbuild.yaml new file mode 100644 index 000000000..61e439fa3 --- /dev/null +++ b/cicd/cloudbuild.yaml @@ -0,0 +1,41 @@ +# +# Author: Hari Sekhon +# Date: 2020-12-19 16:27:26 +0000 (Sat, 19 Dec 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G C P C l o u d B u i l d +# ============================================================================ # + +# References: +# +# https://cloud.google.com/cloud-build/docs/build-config +# +# https://cloud.google.com/cloud-build/docs/build-debug-locally + + +# gcloud builds submit --config cloudbuild.yaml . +# +# cloud-build-local --config cloudbuild.yml --dryrun=false . + +# tars $PWD to bucket called ${PROJECT_ID}_cloudbuild + +timeout: 3660s + +steps: + - name: ubuntu:18.04 + entrypoint: bash + args: + - '-c' + - | + setup/ci_bootstrap.sh && + make build test + timeout: 3600s diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py new file mode 100755 index 000000000..bbd94dbb3 --- /dev/null +++ b/cloudera_navigator_tables_used.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-09 11:35:47 +0000 (Mon, 09 Mar 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Processes Cloudera Navigator API exported CSV logs to list the tables used (SELECT'ed from) + +This allows you to see if you're wasting time maintaining datasets nobody is using + +Handles more than naive filtering delimited column numbers which will miss many table and database names: + + 1. table/database name fields are often blank and need to be inferred from SQL queries field + (currently limited to 'SELECT ... FROM ...' because JOINs are often complicated by use of table aliases) + 2. SQL queries often contain newlines which break the rows up - these are recombined in to single records + 3. multi-line SQL queries have comments stripped out to avoid false positives of what is being used + 4. where table/database field aren't available, also checks if inferrable from resource field + to determine database and table name before parsing SQL which is a last resort + 5. optionally ignore selected users by regex + - matches user or kerberos principal + - eg. to omit ETL service account from skewing data access results + +Supports reading directly from gzipped logs if they end in .gz file extension. +However, the gzip library may have issues around universal newline support, if so, gunzip first. + +See cloudera_navigator_audit_logs_download.sh for a script to export these logs + +./cloudera_navigator_tables_used.py navigator_audit_2019_hive.csv navigator_audit_2019_impala.csv \\ + navigator_audit_2020_hive.csv navigator_audit_2020_impala.csv + +Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison): + +"database","table" + +Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 +(but may require ongoing tweaks depending on quirks in your data set or changes in the API / logs) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import csv +import gzip +import logging +import os +import re +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_regex, isInt, isPythonMinVersion + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.3.0' + + +class ClouderaNavigatorTablesUsed(CLI): + + def __init__(self): + # Python 2.x + super(ClouderaNavigatorTablesUsed, self).__init__() + # Python 3.x + # super().__init__() + csv.field_size_limit(sys.maxsize) + self.delimiter = None + self.quotechar = None + self.escapechar = None + self.timeout_default = None + #self.data = {} + self.indicies = {} + self.len_headers = None + self.table_regex = r'[\w\.`]+' + self.re_table = re.compile(self.table_regex) + # doesn't handle JOINs because SQL pros usually use table aliases + self.re_select_from_table = re.compile(r'\bSELECT\b.+\bFROM\b(?:\s|\n)+({table_regex})'\ + .format(table_regex=self.table_regex), \ + re.I | re.MULTILINE | re.DOTALL) + ignore_statements = [ + 'SHOW', + 'DESCRIBE', + 'USE', + 'CREATE', + 'DROP', + 'INSERT', + 'DELETE', + 'UPDATE', + 'GET_TABLES', + 'GET_SCHEMAS', + 'VIEW_METADATA', + 'ANALYZE', + r'COMPUTE\s+STATS', + 'REFRESH', + r'INVALIDATE\s+METADATA', + ] + self.re_ignore = re.compile(r'^\s*\b(?:' + '|'.join(ignore_statements) + r')\b',\ + re.I | re.MULTILINE | re.DOTALL) + # 2020-01-31T20:45:59.000Z + self.re_timestamp = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$') + self.re_ignored_users = None + self.csv_writer = None + self.operations_to_ignore = [ + '', + 'HIVEREPLICATIONCOMMAND', + 'START', + 'STOP', + 'RESTART', + 'LOAD', + 'SWITCHDATABASE', + 'USE', + ] + + def add_options(self): + super(ClouderaNavigatorTablesUsed, self).add_options() + # must set type to str otherwise csv module gives this error on Python 2.7: + # TypeError: "delimiter" must be string, not unicode + # type=str worked with argparse but when integrated with CLI then 'from __future__ import unicode_literals' + # breaks this - might break in Python 3 if the impyla module doesn't fix behaviour + self.add_opt('-d', '--delimiter', default=',', type=str, help='Delimiter to use for outputting (default: ,)') + self.add_opt('-Q', '--quotechar', default='"', type=str, + help='Generate quoted CSV output (recommended, default is double quote \'"\')') + self.add_opt('-E', '--escapechar', help='Escape char if needed (for both reading and writing)') + self.add_opt('-i', '--ignore-users', help='Users to ignore, comma separated regex values') + + def process_options(self): + super(ClouderaNavigatorTablesUsed, self).process_options() + self.delimiter = self.get_opt('delimiter') + self.quotechar = self.get_opt('quotechar') + self.escapechar = self.get_opt('escapechar') + ignore_users = self.get_opt('ignore_users') + if ignore_users: + ignored_users = ignore_users.split(',') + for username in ignored_users: + validate_regex(username, 'ignored user') + # account for kerberized names - user, user@domain.com or user/host@domain.com + self.re_ignored_users = re.compile('^(?:' + '|'.join(ignored_users) + ')(?:[@/]|$)', re.I) + if not self.args: + self.usage('no CSV file argument given') + + def run(self): + quoting = csv.QUOTE_ALL + if self.quotechar == '': + quoting = csv.QUOTE_NONE + + #fieldnames = ['database', 'table', 'user'] + fieldnames = ['database', 'table'] + self.csv_writer = csv.DictWriter(sys.stdout, + delimiter=self.delimiter, + quotechar=self.quotechar, + escapechar=self.escapechar, + quoting=quoting, + fieldnames=fieldnames) + mode = 'rtU' + if isPythonMinVersion(3): + mode = 'rt' + # open(..., encoding="utf-8") is Python 3 only - uses system default otherwise + for filename in self.args: + if filename.endswith('.gz'): + log.debug("processing gzip'd file: %s", filename) + with gzip.open(filename, mode) as filehandle: + self.process_file(filehandle) + else: + log.debug("processing file: %s", filename) + with open(filename, mode) as filehandle: + self.process_file(filehandle) + + #csv_writer.writeheader() + #for database in sorted(self.data): + # for table in sorted(self.data[database]): + # csv_writer.writerow({'database': database, + # 'table': table}) + #if log.isEnabledFor(logging.DEBUG): + # sys.stdout.flush() + +# Navigator API Audit log output is a mess with duplicate columns and different naming conventions, +# eg. identical SQL in fields 18 and 36 +# table and database names in fields 19+21 vs 40+41 +# all duplicates with different header names +# +# # same result for navigator_audit_2019_impala.csv +# csv_header_indices.sh navigator_audit_2019_hive.csv +# 0 Timestamp +# 1 Username +# 2 "IP Address" +# 3 "Service Name" +# 4 Operation +# 5 Resource +# 6 Allowed +# 7 Impersonator +# 8 sub_operation +# 9 entity_id +# 10 stored_object_name +# 11 additional_info +# 12 collection_name +# 13 solr_version +# 14 operation_params +# 15 service +# 16 operation_text +# 17 url +# 18 operation_text +# 19 table_name +# 20 resource_path +# 21 database_name +# 22 object_type +# 23 Source +# 24 Destination +# 25 Permissions +# 26 "Delegation Token ID" +# 27 "Table Name" +# 28 Family +# 29 Qualifier +# 30 "Operation Text" +# 31 "Database Name" +# 32 "Table Name" +# 33 "Object Type" +# 34 "Resource Path" +# 35 "Usage Type" +# 36 "Operation Text" +# 37 "Query ID" +# 38 "Session ID" +# 39 Status +# 40 "Database Name" +# 41 "Table Name" +# 42 "Object Type" +# 43 Privilege + + def process_file(self, filehandle): + csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') + try: + # Python 2 + headers = csv_reader.next() + except AttributeError: + # Python 3 + headers = next(csv_reader) + self.len_headers = len(headers) + # needed to ensure row joining works later on with number of fields left + assert self.len_headers == 44 + user_index = 1 + operation_index = 4 + resource_index = 5 + object_index = 22 # used by collapse_sql_fields to check if SQL was split, do not change to index 33! + # -- + # with massive queries taking the latter 2 is more likely to succeed, + # possibly because there is a rare and subtle issue in collapse_sql_fields + #table_index = 19 + #database_index = 21 + # or + table_index = 41 + database_index = 40 + # -- + # fields 18 and 36 are identical SQL - need both to collapse rows later + sql_index = 18 + sql_index2 = 36 + #assert headers[table_index] == 'table_name' # index 19 + #assert headers[database_index] == 'database_name' # index 21 + assert headers[table_index] == 'Table Name' # index 41 + assert headers[database_index] == 'Database Name' # index 40 + assert headers[user_index] == 'Username' + assert headers[operation_index] == 'Operation' + assert headers[resource_index] == 'Resource' + assert headers[sql_index] == 'operation_text' + assert headers[sql_index2] == 'Operation Text' + assert headers[object_index] == 'object_type' + self.indicies = { + 'user_index': user_index, + 'operation_index': operation_index, + 'resource_index': resource_index, + 'table_index': table_index, + 'database_index': database_index, + 'sql_index': sql_index, + 'sql_index2': sql_index2, # needed for collapsing rows inflated by SQL fragmentation + 'object_index': object_index + } + self.process_rows(csv_reader) + + # logic to reconstruct rows because the Navigator API breaks the record format + # with newlines in SQL coming out literally and fragmenting the records + def process_rows(self, csv_reader): + last_row = [] + for current_row in csv_reader: + #log.debug('current row = %s', current_row) + if not current_row: + continue + if self.is_new_record(current_row): + row = last_row + last_row = current_row + else: + last_row += current_row + continue + if not row: + continue + self.process_row(row) + self.process_row(last_row) + + # originally did this by counting fields but SQL fragmentation generates extra fields + def is_new_record(self, current_row): + return self.re_timestamp.match(current_row[0]) + + def process_row(self, row): + if not row: + return + log.debug('processing row = %s', row) + len_row = len(row) + log.debug('row len = %s', len_row) + if len_row > self.len_headers: + row = self.collapse_sql_fields(row=row) + len_row = len(row) + if len_row != self.len_headers: + raise AssertionError('row items ({}) != header items ({}) for offending row: {}'\ + .format(len_row, self.len_headers, row)) + (database, table) = self.parse_table(row) + self.output(row=row, database=database, table=table) + + def parse_table(self, row): + #log.debug(row) + user = row[self.indicies['user_index']] + # user: 'hari.sekhon' + # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com' + if self.re_ignored_users and self.re_ignored_users.match(user): + log.debug('skipping row for ignored user %s: %s', user, row) + return (None, None) + database = row[self.indicies['database_index']].strip() + table = row[self.indicies['table_index']].strip() + if not database or not table or not self.re_table.match('{}.{}'.format(database, table)): + #log.info('table not found in fields for row: %s', row) + operation = row[self.indicies['operation_index']] + if operation in self.operations_to_ignore: + return (None, None) + elif operation == 'QUERY': + query = row[self.indicies['sql_index']] + # cheaper than re_ignore to pre-filter + if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'): + return (None, None) + (database, table) = self.get_db_table_from_resource(row) + if database and table: + pass + else: + log.debug('database/table not found in row: %s', row) + log.debug('trying to parse: %s', query) + match = self.re_select_from_table.search(query) + if match: + table = match.group(1) + if '.' in table: + (database, table) = table.split('.', 1) + # could use .search but all these seem to be at beginning + elif self.re_ignore.match(query): + return (None, None) + else: + log.warning('failed to parse database/table from query: %s', query) + return (None, None) + else: + log.debug('database/table not found in row and operation is not a query to parse: %s', row) + return (None, None) + if not table and not database: + return (None, None) + if table: + table = table.lower().strip('`') + if ' ' in table: + raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\ + .format(table, self.index_output(row))) + if database: + database = database.lower().strip('`') + if ' ' in database: + raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\ + .format(database, self.index_output(row))) + if table == 'null': + raise AssertionError('table == null - parsing error for row: {}'.format(row)) + return (database, table) + + def get_db_table_from_resource(self, row): + database = None + table = None + resource = row[self.indicies['resource_index']] + if resource and \ + ':' in resource and \ + 'null:null' not in resource: + # database:table in Resource field + (database, table) = resource.split(':', 1) + return (database, table) + + def output(self, row, database, table): + if not self.re_table.match('{}.{}'.format(database, table)): + log.warning('%s.%s does not match table regex', database, table) + return + # instead of collecting in ram, now just post-process through sort -u + # this way it is easier to see live extractions, --debug and correlate + #self.data[database] = self.data.get(database, {}) + #self.data[database][table] = 1 + if table and not database: + log.info('got table but not database for row: %s', row) + if database and not table: + log.info('got database but not table for row: %s', row) + if not table and not database: + return + #self.csv_writer.writerow({'database': database, 'table': table, 'user': row[self.indicies['user_index']]}) + self.csv_writer.writerow({'database': database, 'table': table}) + if log.isEnabledFor(logging.DEBUG): + sys.stdout.flush() + + def collapse_sql_fields(self, row): + sql_index = self.indicies['sql_index'] + sql_index2 = self.indicies['sql_index2'] + object_index = self.indicies['object_index'] + len_row = len(row) + if len_row > self.len_headers: + log.debug('collapsing fields in row: %s', row) + # divide by 2 to account for this having been done twice in duplicated SQL operational text + # Update: appears this broke as only 2nd occurence of SQL operational text field got split to new fields, + # which is weird because the log shows both 1st and 2nd SQL text fields were double quoted + difference = len_row - self.len_headers + # seems first occurrence doesn't get split in some occurence, + # wasn't related to open in newline universal mode though + # if 2 fields after isn't the /user/hive/warehouse/blah.db then 1st SQL wasn't split + # would have to regex /user/hive/warehouse/blah.db(?:/table)? + #if not row[sql_index+2].endswith('.db'): + # if object field is TABLE or DATABASE then 1st sql field wasn't split + if row[object_index] not in ('TABLE', 'DATABASE'): + difference /= 2 + # slice indicies must be integers + if not isInt(difference): + raise AssertionError("difference in field length '{}' is not an integer for row: {}"\ + .format(difference, row)) + difference = int(difference) + row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index+difference:] + row[sql_index2] = ','.join([self.sql_decomment(_) for _ in row[sql_index2:difference]]) + row = row[:sql_index2] + row[sql_index2+difference:] + log.debug('collapsed row: %s', row) + else: + log.debug('not collapsing row: %s', row) + return row + + @staticmethod + def sql_decomment(string): + return string.split('--')[0].strip() + + @staticmethod + def index_output(obj): + return '\n'.join(['{}\t{}'.format(index, item) for (index, item) in enumerate(obj)]) + + +if __name__ == '__main__': + ClouderaNavigatorTablesUsed().main() diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py new file mode 100755 index 000000000..972716236 --- /dev/null +++ b/cloudera_navigator_tables_used_postgres.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-16 19:21:24 +0000 (Mon, 16 Mar 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Processes Cloudera Navigator CSV logs exported from PostgreSQL to list the tables used (SELECT'ed from) + +This allows you to see if you're wasting time maintaining datasets nobody is using + +See cloudera_navigator_audit_logs_export_postgres.sh for a script to export these logs + +Supports reading directly from gzipped logs if they end in .gz file extension. +However, the gzip library may have issues around universal newline support, if so, gunzip first. + +./cloudera_navigator_tables_used_postgres.py nav.public.hive_audit_events_*.csv.gz \\ + nav.public.impala_audit_events_*.csv.gz ... + +Output is quoted CSV format to stdout (same as hive_schemas_csv.py for easier comparison): + +"database","table" + +Tested on Navigator logs for Hive/Impala on Cloudera Enterprise 5.10 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import csv +#import logging +import os +import re +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +lib = os.path.join(srcdir, 'lib') +sys.path.append(pylib) +sys.path.append(lib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, isInt + #from harisekhon import CLI + from cloudera_navigator_tables_used import ClouderaNavigatorTablesUsed +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.2.1' + + +class ClouderaNavigatorTablesUsedPostgreSQL(ClouderaNavigatorTablesUsed): + + def __init__(self): + # Python 2.x + super(ClouderaNavigatorTablesUsedPostgreSQL, self).__init__() + # Python 3.x + # super().__init__() + # recombine records due to SQL \n breaking up records, new records start like: + # 306529,1574163624392,1,hive, + self.re_new_record = re.compile(r'^\d+,\d+,[01],(?:hive|impala),') + # get db + table from resource path (just one layer of checks) + self.re_resource = re.compile(r'/(\w+)\.db/(\w+)') + +# Navigator table logs: +# +# gzcat nav.public.hive_audit_events_2019_11_19.csv.gz | csv_header_indices.sh +# 0 id +# 1 event_time +# 2 allowed +# 3 service_name +# 4 username +# 5 ip_addr +# 6 operation +# 7 database_name +# 8 object_type +# 9 table_name +# 10 operation_text +# 11 impersonator +# 12 resource_path +# 13 object_usage_type + +# gzcat nav.public.impala_audit_events_2019_11_19.csv.gz | csv_header_indices.sh +# 0 id +# 1 event_time +# 2 allowed +# 3 service_name +# 4 username +# 5 impersonator +# 6 ip_addr +# 7 operation +# 8 query_id +# 9 session_id +# 10 status +# 11 database_name +# 12 object_type +# 13 table_name +# 14 privilege +# 15 operation_text + + def process_file(self, filehandle): + csv_reader = csv.reader(filehandle, delimiter=',', quotechar='"', escapechar='\\') + headers = csv_reader.next() + self.len_headers = len(headers) + # needed to ensure row joining works later on with number of fields left + assert self.len_headers == 14 or self.len_headers == 16 + user_index = 4 + assert headers[user_index] == 'username' + # Hive postgres audit log + if self.len_headers == 14: + operation_index = 6 + database_index = 7 + table_index = 9 + sql_index = 10 + resource_index = 12 + assert headers[resource_index] == 'resource_path' + # Impala postgres audit log + elif self.len_headers == 16: + operation_index = 7 + database_index = 11 + table_index = 13 + sql_index = 15 + resource_index = None + else: + raise AssertionError('headers != 14 or 16 - unrecognized audit log - not Hive or Impala') + assert headers[sql_index] == 'operation_text' + assert headers[database_index] == 'database_name' + assert headers[table_index] == 'table_name' + assert headers[operation_index] == 'operation' + self.indicies = { + 'user_index': user_index, + 'operation_index': operation_index, + 'resource_index': resource_index, + 'table_index': table_index, + 'database_index': database_index, + 'sql_index': sql_index, + } + self.process_rows(csv_reader) + + def is_new_record(self, current_row): + return self.re_new_record.match(','.join(current_row)) + + def parse_table(self, row): + #log.debug(row) + user = row[self.indicies['user_index']] + # user: 'hari.sekhon' + # kerberos principals: 'hari.sekhon@somedomain.com' or 'impala/fqdn@domain.com' + if self.re_ignored_users and self.re_ignored_users.match(user): + log.debug('skipping row for ignored user %s: %s', user, row) + return (None, None) + database = row[self.indicies['database_index']].strip() + table = row[self.indicies['table_index']].strip() + if not database or not table or not self.re_table.match('{}.{}'.format(database, table)): + #log.info('table not found in fields for row: %s', row) + operation = row[self.indicies['operation_index']] + if operation in self.operations_to_ignore: + return (None, None) + elif operation == 'QUERY': + query = row[self.indicies['sql_index']] + # cheaper than re_ignore to pre-filter + if query in ('GET_TABLES', 'GET_SCHEMAS', 'INVALIDATE METADATA'): + return (None, None) + (database, table) = self.get_db_table_from_resource(row) + if database and table: + pass + else: + log.debug('database/table not found in row: %s', row) + log.debug('trying to parse: %s', query) + match = self.re_select_from_table.search(query) + if match: + table = match.group(1) + if '.' in table: + (database, table) = table.split('.', 1) + # could use .search but all these seem to be at beginning + elif self.re_ignore.match(query): + return (None, None) + else: + log.warning('failed to parse database/table from query: %s', query) + return (None, None) + else: + log.debug('database/table not found in row and operation is not a query to parse: %s', row) + return (None, None) + if not table and not database: + return (None, None) + if table: + table = table.lower().strip('`') + if ' ' in table: + raise AssertionError('table \'{}\' has spaces - parsing error for row: {}'\ + .format(table, self.index_output(row))) + if database: + database = database.lower().strip('`') + if ' ' in database: + raise AssertionError('database \'{}\' has spaces - parsing error for row: {}'\ + .format(database, self.index_output(row))) + if table == 'null': + raise AssertionError('table == null - parsing error for row: {}'.format(row)) + return (database, table) + + def get_db_table_from_resource(self, row): + # only available for hive audit logs, not impala + if self.indicies['resource_index'] is None: + return (None, None) + database = None + table = None + resource = row[self.indicies['resource_index']] + if resource: + match = self.re_resource.search(resource) + if match: + database = match.group(1) + table = match.group(2) + return (database, table) + + def collapse_sql_fields(self, row): + sql_index = self.indicies['sql_index'] + len_row = len(row) + if len_row > self.len_headers: + log.debug('collapsing fields in row: %s', row) + difference = len_row - self.len_headers + # slice indicies must be integers + if not isInt(difference): + raise AssertionError("difference in field length '{}' is not an integer for row: {}"\ + .format(difference, row)) + difference = int(difference) + row[sql_index] = ','.join([self.sql_decomment(_) for _ in row[sql_index:difference]]) + row = row[:sql_index] + row[sql_index+difference:] + log.debug('collapsed row: %s', row) + else: + log.debug('not collapsing row: %s', row) + return row + + +if __name__ == '__main__': + ClouderaNavigatorTablesUsedPostgreSQL().main() diff --git a/codefresh.yml b/codefresh.yml new file mode 100644 index 000000000..7cddcaf7c --- /dev/null +++ b/codefresh.yml @@ -0,0 +1,43 @@ +# +# Author: Hari Sekhon +# Date: 2020-02-24 17:43:07 +0000 (Mon, 24 Feb 2020) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# C o d e f r e s h C I +# ============================================================================ # + +# https://codefresh.io/docs/docs/codefresh-yaml/ + +version: "1.0" +stages: + - "checkout" + - "build" +steps: + checkout: + type: "git-clone" + description: "Cloning main repository..." + repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}' + revision: "${{CF_REVISION}}" + stage: "checkout" + build: + title: Running docker image + type: freestyle + working_directory: '${{CF_REPO_NAME}}' + arguments: + image: 'ubuntu:18.04' + commands: + - setup/ci_bootstrap.sh + - make init + - make ci + - make test diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py index 477198dbf..2e675c253 100755 --- a/crunch_accounting_csv_statement_converter.py +++ b/crunch_accounting_csv_statement_converter.py @@ -1,18 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-02-04 23:24:30 +0000 (Mon, 04 Feb 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.3' +__version__ = '0.7.4' class CrunchAccountingCsvStatementConverter(CLI): @@ -107,6 +107,7 @@ def run(self): log.info("converted '%s' => '%s'", filename, target_filename) else: log.error("FAILED to convert filename '%s'", filename) + sys.exit(2) log.info('Final Balance: {}'.format(self.running_balance)) def convert(self, filename, target_filename): @@ -157,7 +158,7 @@ def reverse_contents(filename): return tmp_filename def detect_columns(self, csvreader): - headers = csvreader.next() + headers = next(csvreader) if headers[0][0] == '{': log.error('JSON opening braces detected, not a CSV?') return False @@ -167,17 +168,21 @@ def detect_columns(self, csvreader): # want Transaction Date and not Posted Date if 'Date' in value and not 'Posted' in value: positions['date'] = position - elif 'Merchant Name' in value: - positions['desc'] = position # Original Amount column will be original currency eg 499 USD, but we only want native currency eg. 421.33 elif 'Amount' in value and not 'Original' in value: positions['amount'] = position elif 'Balance' in value: balance_position = position + # Barclaycard CSVs + elif 'Merchant Name' in value: + positions['desc'] = position + # Barclays CSVs + elif 'Memo' in value: + positions['desc'] = position for pos in positions: if positions[pos] is None: log.error('field %s not found', pos) - return False + sys.exit(1) if balance_position is None and self.running_balance is None: self.usage('no balance column detected, please specify --starting-balance') return (positions, balance_position) @@ -198,7 +203,8 @@ def get_csvreader(filename): csvreader = csv.reader(filehandle, dialect) except csv.Error as _: log.warning('file %s: %s', filename, _) - return None + # in Python 2 must be string not unicode + csvreader = csv.reader(filehandle, delimiter=str(','), quotechar=None) csvreader = CrunchAccountingCsvStatementConverter.validate_csvreader(csvreader, filename) filehandle.seek(0) return csvreader @@ -220,11 +226,12 @@ def validate_csvreader(csvreader, filename): # extra protection along the same lines as anti-json: # the first char of field should be alphanumeric, not syntax # however instead of isAlnum allow quotes for quoted CSVs to pass validation - if not isChars(field_list[0][0], 'A-Za-z0-9"'): - log.error('non-alphanumeric / quote opening character detected in CSV') - return None + if field_list[0] not in ("", " ") and not isChars(field_list[0][0], 'A-Za-z0-9"'): + log.warning('non-alphanumeric / quote opening character detected in CSV first field' + \ + '"{}"'.format(field_list[0])) + #return None count += 1 - except csv.Error as _: + except csv.Error as _: log.warning('file %s, line %s: %s', filename, csvreader.line_num, _) return None if count == 0: diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh new file mode 100755 index 000000000..61a1235bd --- /dev/null +++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +. "$srcdir/lib.sh" + +# statements should be named in format: Barclaycard_Statement_YYYY-MM-DD.csv +export STATEMENT_GLOB="Barclaycard_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" + +generate_crunch_statements --credit-card --reverse-order diff --git a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh new file mode 100755 index 000000000..f708fec0c --- /dev/null +++ b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +. "$srcdir/lib.sh" + +# statements should be named in format: Barclays_Statement_YYYY-MM-DD.csv +export STATEMENT_GLOB="Barclays_Statement_[[:digit:]][[:digit:]][[:digit:]][[:digit:]]-[[:digit:]][[:digit:]]-[[:digit:]][[:digit:]].csv" + +# Barclays CSV statements often have whitespace starting fields instead of blank or 'null' +# unfortunately this resets all the original CSV timestamps each run so it's best to do only where needed +# UPDATE: no longer necessary, converter just ignores these blank fields now in the validation +#for statement in $STATEMENT_GLOB; do + #perl -pi -e 's/^\s+,/,/' "$statement" +#done + +generate_crunch_statements --reverse-order diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh new file mode 100755 index 000000000..e77e66834 --- /dev/null +++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1090 +#. "$srcdir/lib/utils.sh" + +converter="$srcdir/../crunch_accounting_csv_statement_converter.py" + +get_latest_crunch_statement(){ + local latest_crunch_statement + for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + latest_crunch_statement="$crunch_statement" + fi + done + if [ -n "${latest_crunch_statement:-}" ]; then + echo "$latest_crunch_statement" + fi +} + +get_final_balance_from_statement(){ + local crunch_statement="$1" + if ! [[ "$crunch_statement" =~ _crunch.csv$ ]]; then + echo "invalid statement passed to get_final_balance_from_statement(), must be *_crunch.csv" >&2 + exit 1 + fi + tail -n 1 "$crunch_statement" | awk -F, '{print $4}' +} + +get_starting_balance(){ + local starting_balance + local latest_crunch_statement="$1" + if [ -n "${latest_crunch_statement:-}" ]; then + echo "latest crunch statement is $latest_crunch_statement" >&2 + starting_balance="$(get_final_balance_from_statement "$latest_crunch_statement")" + else + echo "no latest crunch statement, getting starting balance from environment variable \$STARTING_BALANCE" >&2 + starting_balance="${STARTING_BALANCE:-}" + if [ -z "$starting_balance" ]; then + echo "last crunch statement not found, you must specify the last balance manually via the environment variable \$STARTING_BALANCE" >&2 + exit 1 + fi + fi + echo "starting balance: $starting_balance" >&2 + echo "$starting_balance" +} + +generate_crunch_statements(){ + # only generate statements newer than the last generated one which provides the starting balance + local passed_latest_statement=0 + local latest_crunch_statement + local starting_balance + latest_crunch_statement="$(get_latest_crunch_statement)" + if [ -z "$latest_crunch_statement" ]; then + passed_latest_statement=1 + fi + starting_balance="$(get_starting_balance "$latest_crunch_statement")" + for statement in $STATEMENT_GLOB; do + crunch_statement="${statement%.csv}_crunch.csv" + if [ -f "$crunch_statement" ]; then + if [ $passed_latest_statement = 0 ] && + [ "$crunch_statement" = "$latest_crunch_statement" ]; then + passed_latest_statement=1 + fi + continue + fi + if [ $passed_latest_statement -lt 1 ]; then + continue + fi + "$converter" "$@" --starting-balance "$starting_balance" "$statement" + starting_balance="$(get_final_balance_from_statement "$crunch_statement")" + done +} diff --git a/docker_pull_all_images.sh b/docker_pull_all_images.sh index 430ac70d5..e6d39fd0d 100755 --- a/docker_pull_all_images.sh +++ b/docker_pull_all_images.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/docker_pull_all_images_all_tags.sh b/docker_pull_all_images_all_tags.sh index 3656395cb..d12c3e25b 100755 --- a/docker_pull_all_images_all_tags.sh +++ b/docker_pull_all_images_all_tags.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/docker_pull_all_tags.sh b/docker_pull_all_tags.sh index e555af230..229f3fe68 100755 --- a/docker_pull_all_tags.sh +++ b/docker_pull_all_tags.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/docker_registry_show_tags.py b/docker_registry_show_tags.py index 50d1af743..8c18d3cdc 100755 --- a/docker_registry_show_tags.py +++ b/docker_registry_show_tags.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help improve this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/dockerfiles_check_git_branches.py b/dockerfiles_check_git_branches.py index a7edf5195..662dce3e2 100755 --- a/dockerfiles_check_git_branches.py +++ b/dockerfiles_check_git_branches.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-05-20 20:24:12 +0100 (Fri, 20 May 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # r""" @@ -91,7 +91,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.7.2' +__version__ = '0.7.3' class DockerfileGitBranchCheckTool(CLI): @@ -321,8 +321,8 @@ def check_file(self, filename, branch): def check_dockerfile_arg(self, filename, branch): log.debug('check_dockerfile_arg({0}, {1})'.format(filename, branch)) - branch_base = str(branch).replace('-dev', '') - (branch_base, branch_versions) = self.branch_version(branch) + branch_stripped = str(branch).replace('-dev', '') + (branch_base, branch_versions) = self.branch_version(branch_stripped) with open(filename) as filehandle: version_index = 0 for line in filehandle: diff --git a/dockerfiles_check_git_tags.py b/dockerfiles_check_git_tags.py index 996f787b8..3e0b627bd 100755 --- a/dockerfiles_check_git_tags.py +++ b/dockerfiles_check_git_tags.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-05-20 20:24:12 +0100 (Fri, 20 May 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # r""" diff --git a/dockerhub_search.py b/dockerhub_search.py index cb226e59f..ba7034145 100755 --- a/dockerhub_search.py +++ b/dockerhub_search.py @@ -1,17 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et +# args: harisekhon # # Author: Hari Sekhon # Date: 2016-05-27 13:15:30 +0100 (Fri, 27 May 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help improve this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -20,12 +21,20 @@ Mimics 'docker search' results format but more flexible -Docker CLI doesn't currently support configuring the returned number of search results and always returns 25: +Older Docker CLI didn't support configuring the returned number of search results and always returned 25: https://github.com/docker/docker/issues/23055 Verbose mode will also show a summary for number of results displayed and total number of results available +Caveat: maxes out at 100 results, to iterate for more than that see dockerhub_search.sh + +See also: + + dockerhub_search.sh + +in the DevOps Bash tools repo - https://github.com/HariSekhon/DevOps-Python-tools + """ from __future__ import absolute_import @@ -56,7 +65,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6' +__version__ = '0.6.2' class DockerHubSearch(CLI): @@ -71,7 +80,7 @@ def __init__(self): self.quiet = False def add_options(self): - self.add_opt('-n', '--num', '--limit', default=50, + self.add_opt('-l', '--limit', default=50, type=int, help='Number of results to return (default: 50)') self.add_opt('-q', '--quiet', action='store_true', help='Output only the image names, one per line (useful for shell scripting)') @@ -84,9 +93,9 @@ def run(self): self.quiet = self.get_opt('quiet') term = self.args[0] log.info('term: %s', term) - num = self.get_opt('num') - validate_int(num, 'limit', 1, 1000) - self.print_results(self.args[0], num) + limit = self.get_opt('limit') + validate_int(limit, 'limit', 1, 1000) + self.print_results(self.args[0], limit) def print_results(self, term, limit=None): data = self.search(term, limit) diff --git a/dockerhub_show_tags.py b/dockerhub_show_tags.py index 88c9ddc08..59cd6cb5f 100755 --- a/dockerhub_show_tags.py +++ b/dockerhub_show_tags.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help improve this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -32,6 +32,7 @@ import json import logging import os +import re import sys import traceback import urllib @@ -52,7 +53,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.6.2' +__version__ = '0.6.3' class DockerHubTags(CLI): @@ -79,8 +80,11 @@ def run(self): self.usage('no repos given as args') self.quiet = self.get_opt('quiet') if not self.quiet: + # cheaper but lgtm hassling me, not a security issue but will shut them up print('\nDocker', end='') - if 'registry.hub.docker.com' in self.url_base: + #if 'registry.hub.docker.com' in self.url_base: + # match anchors but I prefer explicit anchor, more intuitive for other generic language coders + if re.match(r'^https://registry\.hub\.docker\.com/', self.url_base): print('Hub') else: print(' Registry: {0}'.format(self.url_base.split('/v2', 1)[0])) diff --git a/find_active_apache_drill.py b/find_active_apache_drill.py index ff4461960..1e89fb676 100755 --- a/find_active_apache_drill.py +++ b/find_active_apache_drill.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_apache_drill2.py b/find_active_apache_drill2.py index 4dcdf8219..364b0eda3 100755 --- a/find_active_apache_drill2.py +++ b/find_active_apache_drill2.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_cassandra.py b/find_active_cassandra.py index 67360ccc3..c16ee217c 100755 --- a/find_active_cassandra.py +++ b/find_active_cassandra.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_elasticsearch.py b/find_active_elasticsearch.py index 42d98d4b8..be449bc79 100755 --- a/find_active_elasticsearch.py +++ b/find_active_elasticsearch.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Tue Sep 5 10:49:49 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_hadoop2_namenode.py b/find_active_hadoop2_namenode.py index bdb7ee5e4..7aad7e77d 100755 --- a/find_active_hadoop2_namenode.py +++ b/find_active_hadoop2_namenode.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Tue Sep 5 10:49:49 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_hadoop_namenode.py b/find_active_hadoop_namenode.py index ff578d9ef..1f6f2d44d 100755 --- a/find_active_hadoop_namenode.py +++ b/find_active_hadoop_namenode.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Tue Sep 5 10:49:49 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_hadoop_yarn_resource_manager.py b/find_active_hadoop_yarn_resource_manager.py index d1179c7cc..f2f0e48e3 100755 --- a/find_active_hadoop_yarn_resource_manager.py +++ b/find_active_hadoop_yarn_resource_manager.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 6 14:44:38 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_hbase_master.py b/find_active_hbase_master.py index e89c87613..cc7de0222 100755 --- a/find_active_hbase_master.py +++ b/find_active_hbase_master.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_hbase_stargate.py b/find_active_hbase_stargate.py index c62b1756c..79dc122fa 100755 --- a/find_active_hbase_stargate.py +++ b/find_active_hbase_stargate.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_hbase_thrift.py b/find_active_hbase_thrift.py index a83594730..06e44a414 100755 --- a/find_active_hbase_thrift.py +++ b/find_active_hbase_thrift.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_impala.py b/find_active_impala.py index cce6eb399..ae086f46a 100755 --- a/find_active_impala.py +++ b/find_active_impala.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_impala_catalog.py b/find_active_impala_catalog.py index 7fc7a143e..cb6ea0de3 100755 --- a/find_active_impala_catalog.py +++ b/find_active_impala_catalog.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_impala_statestore.py b/find_active_impala_statestore.py index 3ec124494..049ed78f5 100755 --- a/find_active_impala_statestore.py +++ b/find_active_impala_statestore.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_kubernetes_api.py b/find_active_kubernetes_api.py index 0ef8165f8..a93b0bc81 100755 --- a/find_active_kubernetes_api.py +++ b/find_active_kubernetes_api.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_oozie.py b/find_active_oozie.py index 21b9271d6..156bb51a9 100755 --- a/find_active_oozie.py +++ b/find_active_oozie.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_presto_coordinator.py b/find_active_presto_coordinator.py index 13c8f6546..9b0050a47 100755 --- a/find_active_presto_coordinator.py +++ b/find_active_presto_coordinator.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Wed Sep 13 13:58:21 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_active_server.py b/find_active_server.py index edcf0fb49..30810a448 100755 --- a/find_active_server.py +++ b/find_active_server.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-09-29 15:00:36 +0100 (Thu, 29 Sep 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # r""" @@ -96,7 +96,7 @@ See also Advanced HAProxy configurations (part of the Advanced Nagios Plugins Collection) at: - https://github.com/harisekhon/haproxy-configs + https://github.com/HariSekhon/HAProxy-configs """ @@ -112,14 +112,20 @@ import subprocess import sys #from threading import Thread -from multiprocessing.pool import ThreadPool, cpu_count +from multiprocessing.pool import ThreadPool +from multiprocessing import cpu_count # prefer blocking semantics of que.get() rather than handling deque.popleft() => 'IndexError: pop from an empty deque' #from collections import deque -import Queue import traceback from random import shuffle +# Python 2 Queue vs Python 3 queue module :-/ +if sys.version[0] == '2': + import Queue as queue # pylint: disable=import-error +else: + import queue as queue # pylint: disable=import-error try: - import requests + # false positive from pylint, queue is imported first + import requests # pylint: disable=wrong-import-order except ImportError: print(traceback.format_exc(), end='') sys.exit(4) @@ -137,7 +143,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.4' +__version__ = '0.8.6' class FindActiveServer(CLI): @@ -156,7 +162,7 @@ def __init__(self): self.request_timeout = None self.default_num_threads = min(cpu_count() * 4, 100) self.num_threads = None - self.queue = Queue.Queue() + self.queue = queue.Queue() self.pool = None def add_options(self): diff --git a/find_active_solrcloud.py b/find_active_solrcloud.py index b6683fafb..659eceb59 100755 --- a/find_active_solrcloud.py +++ b/find_active_solrcloud.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: Tue Sep 5 10:49:49 CEST 2017 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/find_duplicate_files.py b/find_duplicate_files.py index 00b45fe2e..4c452b03f 100755 --- a/find_duplicate_files.py +++ b/find_duplicate_files.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-08-14 09:50:03 +0100 (Sun, 14 Aug 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -33,12 +33,17 @@ 4. regex capture matching portion - specify a regex to match against the filenames with capture (brackets) and the captured portion will be compared among files. If no capture brackets are detected then will treat the entire regex as the capture. - Regex is case insensitive by default + Regex is case insensitive by default and applies only to the file's basename + +Exits with exit code 4 if duplicates are found Can restrict methods of finding duplicates to any combination of --name / --size / --checksum (checksum implies size as an efficiency shortcut) / --regex. If none are specified then will try name, size + checksum. If specifying any one of these options then the others will not run unless also explicitly specified. +If you want to find files that are probably the same by byte count but may not have the same checksum due to minor +corruption, such as large media files, then specify --size but do not specify --checksum which supercedes it + Caveats: - The limitation of the checksum approach is that it can't determine files as duplicates if there is any @@ -47,10 +52,12 @@ - By default this program will short-circuit to stop processing a file as soon as it is determined to be a duplicate file via one of the above methods in that order for efficiency. This means that if 2 files have duplicate names, and a third has a different name but the same checksum as the second one, the second one's size + checksum will not have -been recorded stored and so the third duplicate will not be detected. However, if you removed one duplicate the next -run of this program would find the other duplicate via the other dimension of checking. Given it's a rare condition -it's probably not worth the extra overhead in everyday use but this behaviour can be overridden by specifying the ---no-short-circuit option too run every check on every file. Be aware this will slow down the process. +been checked and so a third duplicate with a different name will not be detected by size / checksum. In most cases this +is a good thing to finish quicker and avoid unnecessary checksumming which is computationally expensive and time +consuming for large files. If you remove one duplicate then the next run of this program would find the other +duplicate via the additional checks of size and checksumming. Given it's a rare condition it's probably not worth the +extra overhead in everyday use but this behaviour can be overridden by specifying the --no-short-circuit option to run +every check on every file. Be aware this will slow down the process. To see progress of which files are matching size, backtracking to hash them for comparison etc use --verbose twice or -vv. To see which files are being checked use triple verbose mode -vvv @@ -81,7 +88,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.4' +__version__ = '0.6.2' class FindDuplicateFiles(CLI): @@ -165,7 +172,7 @@ def process_args(self): log_option('compare by name', self.compare_by_name) log_option('compare by size', self.compare_by_size) log_option('compare by checksum', self.compare_by_checksum) - log_option('compare by regex', True if self.regex else False) + log_option('compare by regex', bool(self.regex)) return args @staticmethod @@ -210,36 +217,36 @@ def run(self): for filepath in sorted(self.dup_filepaths): print(filepath) sys.exit(4) - print('Duplicates detected!\n') + print('# Duplicates detected!') if self.dups_by_name: - print('Duplicates by name:\n') + print('\n# Duplicates by name:\n') for basename in self.dups_by_name: - print("--\nbasename '{0}':".format(basename)) + print("# --\n# basename '{0}':".format(basename)) for filepath in sorted(self.dups_by_name[basename]): print(filepath) if self.dups_by_size: - print('Duplicates by size:\n') + print('\n# Duplicates by size:\n') for size in self.dups_by_size: - print("--\nsize '{0}' bytes:".format(size)) + print("# --\n# size '{0}' bytes:".format(size)) for filepath in sorted(self.dups_by_size[size]): print(filepath) if self.dups_by_hash: - print('Duplicates by checksum:\n') + print('\n# Duplicates by checksum:\n') for checksum in self.dups_by_hash: - print("--\nchecksum '{0}':".format(checksum)) + print("# --\n# checksum '{0}':".format(checksum)) for filepath in sorted(self.dups_by_hash[checksum]): print(filepath) if self.dups_by_regex: - print('Duplicates by regex match ({0}):\n'.format(self.regex)) + print('\n# Duplicates by regex match ({0}):\n'.format(self.regex)) for matching_portion in self.dups_by_regex: - print("--\nregex matching portion '{0}':".format(matching_portion)) + print("# --\n# regex matching portion '{0}':".format(matching_portion)) for filepath in sorted(self.dups_by_regex[matching_portion]): print(filepath) sys.exit(4) elif self.failed: sys.exit(2) else: - print('No Duplicates Found') + print('# No Duplicates Found') sys.exit(0) # def check_path(self, path): @@ -297,6 +304,7 @@ def check_path(self, path): def is_file_dup(self, filepath): log.debug("checking file path '%s'", filepath) + # pylint: disable=no-else-return if os.path.islink(filepath): log.debug("ignoring symlink '%s'", filepath) return False @@ -340,8 +348,7 @@ def is_file_dup_by_name(self, filepath): self.dups_by_name[basename].add(self.files[basename]) self.dups_by_name[basename].add(filepath) return True - else: - self.files[basename] = filepath + self.files[basename] = filepath return False def is_file_dup_by_size(self, filepath): @@ -397,20 +404,21 @@ def is_file_dup_by_hash(self, filepath): return False def is_file_dup_by_regex(self, filepath): - match = re.search(self.regex, filepath) + #match = re.search(self.regex, filepath) + basename = os.path.basename(filepath) + match = re.search(self.regex, basename) if match: log.debug("regex matched file '%s'", filepath) if match.groups(): capture = match.group(1) - if capture in self.regex_captures: - self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set()) - self.dups_by_regex[capture].add(self.regex_captures[capture]) - self.dups_by_regex[capture].add(filepath) - return True - else: - self.regex_captures[capture] = filepath else: - log.error('no capture detected! Did you forget to specify the (brackets) to capture in the regex?') + capture = match.group(0) + if capture in self.regex_captures: + self.dups_by_regex[capture] = self.dups_by_regex.get(capture, set()) + self.dups_by_regex[capture].add(self.regex_captures[capture]) + self.dups_by_regex[capture].add(filepath) + return True + self.regex_captures[capture] = filepath return False diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py new file mode 100755 index 000000000..f982cbddd --- /dev/null +++ b/find_missing_files_in_sequence.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-07-31 11:03:17 +0100 (Fri, 31 Jul 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Finds missing files by numeric sequence, assuming a uniformly numbered file naming convention across files + +Files / directories are given as arguments or via standard input + +Directories are recursed and their files examined for missing numbers before each one + +Only supply files / directories that should be sharing a contiguously numbered file naming convention in each +single run of this tool + +Accounts for zero padding in numbered files + +Caveats: + +- This is more complicated than you'd first think as there are so many file naming variations that no code could ever + be universally bulletproof and will likely require advanced regex tuning to match your use case and naming convention + +- Won't detect missing files higher than the highest numbered file as there is no way to know how many there should be. + If you are looking for missing MP3 files, then you might be able to check the mp3 tag metadata using programs like + 'mediainfo' to get the total number of tracks and see if the files go that high + +- Returns globs by default instead of explicit missing filenames since suffixes can vary after numbers. If you have a + simple enough use case with a single fixed filename convention such as 'blah_01.txt' then you can find code to print + the missing files more explicitly, but in the general case you cannot account for suffix naming that isn't consistent, + such as chapters of audiobooks eg. + + 'blah 01 - chapter about X.mp3' + 'blah 02 - chapter about Y.mp3' + + so in the general case you cannot always infer suffixes, hence why it is left as globs. If you are sure that the + suffixes don't change then you can specify --fixed-suffix and it will infer each file's suffix as the basis for any + numerically missing files in the sequence, but if used where this is not the case, it'll generate a lot of false + positives that the default globbing mode would have handled + +- Doesn't currently find entire missing CD / disks in the naming format, but you should be able to see those cases + easily by eye + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import glob +#import logging +import os +import re +import sys +import traceback +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + from harisekhon.utils import log, log_option, validate_regex, isInt, UnknownError + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.3.2' + + +# pylint: disable=too-many-instance-attributes +class FindMissingFiles(CLI): + + def __init__(self): + # Python 2.x + super(FindMissingFiles, self).__init__() + # Python 3.x + # super().__init__() + self.paths = [] + self.regex_default = r'(? 1: + file_number = self.determine_missing_file_backfill(file_prefix, file_number, padding, file_suffix) + if self.missing_files: + print('\n'.join(reversed(self.missing_files))) + self.missing_files = [] + + def determine_missing_file_backfill(self, file_prefix, file_number, padding, file_suffix): + file_number -= 1 + if self.fixed_suffix: + explicit_last_filename = '{}{:0>%(padding)s}{}' % {'padding': padding} + explicit_last_filename = explicit_last_filename.format(file_prefix, file_number, file_suffix) + if not os.path.isfile(explicit_last_filename): + self.missing_files.append(explicit_last_filename) + else: + file_number = -1 + else: + expected_last_filename_glob = '{}{:0>%(padding)s}*' % locals() + expected_last_filename_glob = expected_last_filename_glob.format(file_prefix, file_number) + if not glob.glob(expected_last_filename_glob): + self.missing_files.append(expected_last_filename_glob) + else: + file_number = -1 + return file_number + + +if __name__ == '__main__': + FindMissingFiles().main() diff --git a/gcp_cloud_function_ifconfig/.gcloudignore b/gcp_cloud_function_ifconfig/.gcloudignore new file mode 100644 index 000000000..42de3fd00 --- /dev/null +++ b/gcp_cloud_function_ifconfig/.gcloudignore @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore + +.git +.gcloudignore +deploy.sh +test/ +tests/ diff --git a/gcp_cloud_function_ifconfig/Makefile b/gcp_cloud_function_ifconfig/Makefile new file mode 100644 index 000000000..656b78879 --- /dev/null +++ b/gcp_cloud_function_ifconfig/Makefile @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Kubernetes-templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +SHELL = /usr/bin/env bash + +.PHONY: default +default: deploy + @: + +.PHONY: deploy +deploy: + @./deploy.sh diff --git a/gcp_cloud_function_ifconfig/README.md b/gcp_cloud_function_ifconfig/README.md new file mode 100644 index 000000000..0485fcf08 --- /dev/null +++ b/gcp_cloud_function_ifconfig/README.md @@ -0,0 +1,45 @@ +Google Cloud Function - ifconfig +===================== + +Queries http://ifconfig.co from GCF to check the routing and external IP being used eg. for comparison with Cloudflare / Firewall rules + +- `main.py` - the code +- `requirements.txt` - the pip modules to bootstrap +- `deploy.sh` - upload the code and deps + +Response is HTTP status code and message, then the raw JSON results + +``` +200 OK + +{ + "ip": "1.2.3.4", + "ip_decimal": 1234567890, + "country": "United States", + "country_iso": "US", + "country_eu": false, + "latitude": 37.751, + "longitude": -97.822, + "time_zone": "America/Chicago", + "asn": "AS15169", + "asn_org": "GOOGLE", + "hostname": "ipv6.gae.googleusercontent.com", + "user_agent": { + "product": "python-requests", + "version": "2.24.0", + "raw_value": "python-requests/2.24.0" + } +} +``` + +For IPv6 the format will be more like: +``` + "ip": "1234:5678:9012:34::a", + "ip_decimal": 12345678901234567890123456789012345678, +``` + +Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: + +``` +./deploy.sh +``` diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh new file mode 100755 index 000000000..39aeba85e --- /dev/null +++ b/gcp_cloud_function_ifconfig/deploy.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cd "$srcdir" + +name="ifconfig" + +# Cloud Functions not available in all regions yet: +# +# https://cloud.google.com/functions/docs/locations +# +# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment +region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" + +# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com +# for serverless VPC access to resources using their Private IPs +# since we're only accessing the SQL Admin API we don't need this +#vpc_connector="my-vpc-connector" + +opts=() +if [ -n "${vpc_connector:-}" ]; then + # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules + opts+=(--vpc-connector "$vpc_connector" --egress-settings all) +fi + +set -x +gcloud functions deploy "$name" --trigger-http \ + --security-level=secure-always \ + --runtime python39 \ + --entry-point main \ + --memory 128MB \ + --region "$region" \ + --timeout 60 \ + "${opts[@]}" \ + --quiet # don't prompt to --allow-unauthenticated + #--max-instances 1 \ diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py new file mode 100755 index 000000000..c9e50b0e7 --- /dev/null +++ b/gcp_cloud_function_ifconfig/main.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access +routing via specified VPC Network to using default NAT Gateway + +Example usage: check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules + +The HTTP request is irrelevant, just pass an empty JSON document '{}', although we could pass the website to query +in a field in which case this would just act as a proxy. + +See Also: gcp_cloud_function_proxy/main.py + + +Tested on GCP Cloud Functions with Python 3.9 + +""" + +# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python + +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python + +import requests + +def main(_): + """Responds to any HTTP request. + Args: + request (flask.Request): HTTP request object. + Returns: + The response text or any set of values that can be turned into a + Response object using + `make_response `. + """ + req = requests.get('http://ifconfig.co/json') # show our IP information for debugging VPC connector routing + status_code = req.status_code + status_message = req.reason + content = req.text + return "{} {}\n\n{}".format(status_code, status_message, content) diff --git a/gcp_cloud_function_ifconfig/requirements.txt b/gcp_cloud_function_ifconfig/requirements.txt new file mode 100644 index 000000000..775d29ac6 --- /dev/null +++ b/gcp_cloud_function_ifconfig/requirements.txt @@ -0,0 +1,2 @@ +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python +requests==2.24.0 diff --git a/gcp_cloud_function_proxy/.gcloudignore b/gcp_cloud_function_proxy/.gcloudignore new file mode 100644 index 000000000..42de3fd00 --- /dev/null +++ b/gcp_cloud_function_proxy/.gcloudignore @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore + +.git +.gcloudignore +deploy.sh +test/ +tests/ diff --git a/gcp_cloud_function_proxy/Makefile b/gcp_cloud_function_proxy/Makefile new file mode 100644 index 000000000..656b78879 --- /dev/null +++ b/gcp_cloud_function_proxy/Makefile @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Kubernetes-templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +SHELL = /usr/bin/env bash + +.PHONY: default +default: deploy + @: + +.PHONY: deploy +deploy: + @./deploy.sh diff --git a/gcp_cloud_function_proxy/README.md b/gcp_cloud_function_proxy/README.md new file mode 100644 index 000000000..1a4c2465a --- /dev/null +++ b/gcp_cloud_function_proxy/README.md @@ -0,0 +1,27 @@ +Google Cloud Function - Proxy +===================== + +Queries a given URL from GCF to check connectivity eg. for testing with Cloudflare / Firewall rules + +Query content: +``` +{"url": "http://ifconfig.co/json"} +``` + +Response is HTTP status code and message, blank line and then the content: + +``` +200 OK + + +``` + +- `main.py` - the code +- `requirements.txt` - the pip modules to bootstrap +- `deploy.sh` - upload the code and deps + +Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: + +``` +./deploy.sh +``` diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh new file mode 100755 index 000000000..d8e04e9b5 --- /dev/null +++ b/gcp_cloud_function_proxy/deploy.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cd "$srcdir" + +name="proxy" + +# Cloud Functions not available in all regions yet: +# +# https://cloud.google.com/functions/docs/locations +# +# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment +region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" + +# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com +# for serverless VPC access to resources using their Private IPs +# since we're only accessing the SQL Admin API we don't need this +#vpc_connector="my-vpc-connector" + +opts=() +if [ -n "${vpc_connector:-}" ]; then + # routes all traffic through VPC connector to re-use the VPC's Cloud NAT IP eg. for permitting in firewall rules + opts+=(--vpc-connector "$vpc_connector" --egress-settings all) +fi + +set -x +gcloud functions deploy "$name" --trigger-http \ + --security-level=secure-always \ + --runtime python39 \ + --entry-point main \ + --memory 128MB \ + --region "$region" \ + --timeout 60 \ + "${opts[@]}" \ + --quiet # don't prompt to --allow-unauthenticated + #--max-instances 1 diff --git a/gcp_cloud_function_proxy/main.py b/gcp_cloud_function_proxy/main.py new file mode 100755 index 000000000..8680d455a --- /dev/null +++ b/gcp_cloud_function_proxy/main.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +GCP Cloud Function to query ifconfig.co to show our IP information for debugging VPC Connector access +routing via specified VPC Network to using default NAT Gateway + +Example usage: + +Check GCF source IP to compare if it's permitted through Cloudflare / Firewall rules + +Test request examples: + + { "url": "http://ifconfig.co/json" } + +defaults to http:// if not specified: + + { "url": "ifconfig.co/json" } + + +Tested on GCP Cloud Functions with Python 3.9 + +""" + +# https://cloud.google.com/functions/docs/writing/http#writing_http_content-python + +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python + +import json +import requests + +def main(request): + """Responds to any HTTP request. + Args: + request (flask.Request): HTTP request object. + Returns: + The response text or any set of values that can be turned into a + Response object using + `make_response `. + """ + data = json.loads(request.data) + url = data['url'] + if '://' not in url: + url = 'http://' + url + req = requests.get(url) + status_code = req.status_code + status_message = req.reason + content = req.text + return "{} {}\n\n{}".format(status_code, status_message, content) diff --git a/gcp_cloud_function_proxy/requirements.txt b/gcp_cloud_function_proxy/requirements.txt new file mode 100644 index 000000000..775d29ac6 --- /dev/null +++ b/gcp_cloud_function_proxy/requirements.txt @@ -0,0 +1,2 @@ +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python +requests==2.24.0 diff --git a/gcp_cloud_function_sql_export/.gcloudignore b/gcp_cloud_function_sql_export/.gcloudignore new file mode 100644 index 000000000..42de3fd00 --- /dev/null +++ b/gcp_cloud_function_sql_export/.gcloudignore @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2020-10-16 11:44:51 +0100 (Fri, 16 Oct 2020) +# +# vim:ts=4:sts=4:sw=4:et +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# see also: massive generic .gcloudignore at https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.gcloudignore + +.git +.gcloudignore +deploy.sh +test/ +tests/ diff --git a/gcp_cloud_function_sql_export/Makefile b/gcp_cloud_function_sql_export/Makefile new file mode 100644 index 000000000..656b78879 --- /dev/null +++ b/gcp_cloud_function_sql_export/Makefile @@ -0,0 +1,22 @@ +# +# Author: Hari Sekhon +# Date: 2021-01-18 18:15:39 +0000 (Mon, 18 Jan 2021) +# +# vim:ts=4:sts=4:sw=4:noet +# +# https://github.com/HariSekhon/Kubernetes-templates +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +SHELL = /usr/bin/env bash + +.PHONY: default +default: deploy + @: + +.PHONY: deploy +deploy: + @./deploy.sh diff --git a/gcp_cloud_function_sql_export/README.md b/gcp_cloud_function_sql_export/README.md new file mode 100644 index 000000000..db3ff61ad --- /dev/null +++ b/gcp_cloud_function_sql_export/README.md @@ -0,0 +1,52 @@ +Google Cloud Function - SQL Backup Exporter to GCS +===================== + +Triggers GCP [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage). + +Solution documentation: + +https://cloud.google.com/solutions/scheduling-cloud-sql-database-exports-using-cloud-scheduler + +- `main.py` - the code +- `requirements.txt` - the pip modules to bootstrap +- `deploy.sh` - upload the code and deps + +Upload the function to GCF in the current GCP project - this script will call `gcloud functions deploy` with the required switches: + +``` +./deploy.sh +``` + +### Solution Dependencies + +- a [Cloud PubSub](https://cloud.google.com/pubsub) topic +- [Cloud Scheduler](https://cloud.google.com/scheduler) jobs to trigger backups + - see `gcp_cloud_schedule_sql_exports.sh` in [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo +- a service account with permissions to access [Cloud SQL](https://cloud.google.com/sql) + - see `gcp_sql_create_readonly_service_account.sh` in [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo +- each [Cloud SQL](https://cloud.google.com/sql) instance to be backed up requires objectCreator permissions to the [GCS](https://cloud.google.com/storage) bucket + - see `gcp_sql_grant_instances_gcs_object_creator.sh` in [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo + +### Serverless Framework + +Instead of `deploy.sh` you can alternatively use the [Serverless](https://www.serverless.com/) framework for which a `serverless.yml` config is provided: + +``` +serverless deploy +``` + +If this is your first time using Serverless then you'll need to install the GCP plugin: + +``` +serverless plugin install --name serverless-google-cloudfunctions +``` + +The `serverless.yml` config expects to find `$GOOGLE_PROJECT_ID` and `$GOOGLE_REGION` environment variables. + +Serverless requires additional permissions for the service account: Deployment Manager Editor and Storage Admin to create deployments and staging buckets. + +You can also build a serverless artifact to `.serverless/` without deploying it (generates Google [Deployment Manager](https://cloud.google.com/deployment-manager) templates and a zip file - useful to check what would be uploaded / ignored): + +``` +serverless package +``` diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh new file mode 100755 index 000000000..c5c3c3249 --- /dev/null +++ b/gcp_cloud_function_sql_export/deploy.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-10-16 10:12:26 +0100 (Fri, 16 Oct 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cd "$srcdir" + +# needed to define the $service_account further down +project="${CLOUDSDK_CORE_PROJECT:-$(gcloud config list --format="value(core.project)")}" + +# Cloud Functions not available in all regions yet: +# +# https://cloud.google.com/functions/docs/locations +# +# gcloud functions deploy doesn't seem to infer CLOUDSDK_COMPUTE_REGION from environment +region="$(gcloud config list --format="value(compute.region)" 2>&1 || :)" +region="${CLOUDSDK_COMPUTE_REGION:-${region:-europe-west1}}" + +name="cloud-sql-backups" +topic="cloud-sql-backups" +service_account="cloud-function-sql-backup@$project.iam.gserviceaccount.com" +# https://console.cloud.google.com/marketplace/product/google/vpcaccess.googleapis.com +# for serverless VPC access to resources using their Private IPs +# since we're only accessing the SQL Admin API we don't need this +#vpc_connector="cloud-sql-backups" + +gcloud functions deploy "$name" --trigger-topic "$topic" \ + --runtime python37 \ + --entry-point main \ + --service-account "$service_account" \ + --region "$region" \ + --memory 128MB \ + --timeout 60 + # may want multiple concurrent calls to different SQL instances at the same time + # + # also doesn't prevent: + # + # "Operation failed because another operation was already in progress." + # + #--max-instances 1 # this isn't good enough because it sets off an async API call, so successive calls can fail if called before that SQL Admin API export has finished + + # --vpc-connector "$vpc_connector" diff --git a/gcp_cloud_function_sql_export/main.py b/gcp_cloud_function_sql_export/main.py new file mode 100755 index 000000000..6e3765667 --- /dev/null +++ b/gcp_cloud_function_sql_export/main.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-10-14 15:29:38 +0100 (Wed, 14 Oct 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +GCP Cloud Function to export a given Cloud SQL database to GCS via PubSub notifications from Cloud Scheduler + +Solution Documentation: + + https://cloud.google.com/solutions/scheduling-cloud-sql-database-exports-using-cloud-scheduler + +GCP Cloud PubSub should be sent payloads like this by Cloud Scheduler (replacing the env vars with your literals): + +{ + "project": "${GOOGLE_PROJECT_ID}", + "instance": "${SQL_INSTANCE_HOST}", + "database": "${DATABASE}", + "bucket": "${BUCKET_NAME}" +} + +Tested on GCP Cloud Functions with Python 3.7 + +""" + +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python + +# Code below is based on solution sample code from the link above + +#import os +import base64 +import logging +import json + +from datetime import datetime +from httplib2 import Http + +from googleapiclient import discovery +from googleapiclient.errors import HttpError +from oauth2client.client import GoogleCredentials + + +# pylint: disable=unused-argument +def main(event, context): +# if os.getenv("DEBUG"): +# # debug level logs don't appear in function details logs tab even with +# # DEBUG=1 runtime env var set and logs severity filter set to >= DEBUG +# #logging.debug('event: %s', event) +# #logging.debug('context: %s', context) +# logging.info('event: %s', event) +# logging.info('context: %s', context) + data = json.loads(base64.b64decode(event['data']).decode('utf-8')) + credentials = GoogleCredentials.get_application_default() + + service = discovery.build('sqladmin', 'v1beta4', http=credentials.authorize(Http()), cache_discovery=False) + + project = data['project'] + bucket = data['bucket'] + bucket = bucket.lstrip('gs://') + instance = data['instance'] + database = data['database'] + timestamp = datetime.now().strftime("%Y-%m-%d_%H%M") + + # .gz extension so it is auto-compressed, saving storage space + billing + backup_uri = "gs://{bucket}/backups/sql/{instance}--{database}--{timestamp}.sql.gz".format( + bucket=bucket, + instance=instance, + database=database, + timestamp=timestamp) + + instances_export_request_body = { + "exportContext": { + "kind": "sql#exportContext", + "fileType": "SQL", + "uri": backup_uri, + "databases": [ + database + ] + } + } + + try: + logging.info("Requesting project '%s' database instance '%s' runs a backup export to bucket '%s' path '%s'", + project, + instance, + bucket, + backup_uri) + request = service.instances().export( + project=project, + instance=instance, + body=instances_export_request_body + ) + response = request.execute() + except HttpError as err: + logging.error("Backup FAILED. Reason: %s", err) + else: + logging.info("Backup Task triggered: %s", response) diff --git a/gcp_cloud_function_sql_export/requirements.txt b/gcp_cloud_function_sql_export/requirements.txt new file mode 100644 index 000000000..2f7349f4d --- /dev/null +++ b/gcp_cloud_function_sql_export/requirements.txt @@ -0,0 +1,3 @@ +# https://cloud.google.com/functions/docs/writing/specifying-dependencies-python +google-api-python-client==1.12.5 +oauth2client==4.1.3 diff --git a/gcp_cloud_function_sql_export/serverless.yml b/gcp_cloud_function_sql_export/serverless.yml new file mode 100644 index 000000000..a73986efb --- /dev/null +++ b/gcp_cloud_function_sql_export/serverless.yml @@ -0,0 +1,137 @@ +# vim:ts=2:sts=2:sw=2:et +# +# Author: Hari Sekhon +# Date: 2020-10-21 11:19:06 +0100 (Wed, 21 Oct 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S e r v e r l e s s +# ============================================================================ # + +# Deploys the Google Cloud Function with a name -- +# +# eg. cloud-sql-backups-production-main + +# Requires the same permissions as the simpler adjacent deploy.sh script but also: +# +# Deployment Manager Editor +# Storage Admin (Storage Object Admin is not enough as it needs to create staging buckets and you'll get weird errors otherwise) +# +# for the credential file's service account which is also used to run the Cloud Function + +# Check generated config after environment variable interpolation: +# +# cd "$(dirname $0)" && serverless print +# +# My advanced vimrc has a hotkey for this: +# +# https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.vimrc + +frameworkVersion: '2' +plugins: + - serverless-google-cloudfunctions + +# to prevent: +# +# Serverless: Configuration warning: Unrecognized provider 'google' +# Serverless: +# Serverless: You're relying on provider plugin which doesn't provide a validation schema for its config. +configValidationMode: off +# after the GCP plugin matures, switch to +#configValidationMode: error + +service: cloud-sql-backups +provider: + name: google + stage: production + runtime: python37 + region: ${env:GOOGLE_REGION, "europe-west1"} + project: ${env:GOOGLE_PROJECT_ID} + + # https://serverless.com/framework/docs/providers/google/guide/credentials/ + # + # path to credentials file needs to be absolute + # + # download a credentials file for the service account: + # + # https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-gcloud + # + # mkdir -pv ~/.gcloud && gcloud iam service-accounts keys create ~/.gcloud/cloud-function-sql-backup-keyfile.json --iam-account "cloud-function-sql-backup@$(gcloud config list --format="get(core.project)").iam.gserviceaccount.com" + # + credentials: ~/.gcloud/cloud-function-sql-backup-keyfile.json + # + # or use personal creds - see $GOOGLE_APPLICATION_CREDENTIALS or generate: + # + # gcloud auth application-default login # [ --client-id-file=~/.gcloud/keyfile.json ] + # + #credentials: ~/.config/gcloud/application_default_credentials.json + #credentials: ${env:HOME}/.config/gcloud/application_default_credentials.json + #credentials: /Users/harisekhon/.config/gcloud/application_default_credentials.json + +# https://www.serverless.com/framework/docs/providers/google/guide/packaging/ +package: + exclude: + ## needs more granular excluding in production as only the serverless provider npm + ## package should be excluded (and not the whole node_modules directory) + #- node_modules/** + #- deploy.sh + #- test/** + #- tests/** + #- .gitignore + #- .gcloudignore + #- .git/** + #- package.json + #- package-lock.json + #- README.md + # XXX: exclude all hidden files and directories + - .* + # XXX: more robust to only whitelist include as per .dockerignore best practice too (unfortunately not supported in .gcloudignore) + - ./** + include: + - main.py + - requirements.txt + # or specify your own zipfile and skip packaging: + #artifact: path/to/my-artifact.zip + +# https://www.serverless.com/framework/docs/providers/google/guide/functions/ +# +# use an array of includes for bigger serverless deployments of multiple functions +#functions: +# - ${file(../foo-functions.yml)} +# - ${file(../bar-functions.yml)} +# +functions: + #myfunc: + # handler: http + # events: + # - http: path + # NOTE: the following uses an "event" event (pubSub event in this case). + # Please create the corresponding resources in the Google Cloud + # before deploying this service through Serverless + main: + handler: main + memorySize: 128 + timeout: 60s + events: + - event: + eventType: providers/cloud.pubsub/eventTypes/topic.publish + #resource: projects/*/topics/my-topic + resource: projects/${env:GOOGLE_PROJECT_ID}/topics/cloud-sql-backups +# you can define resources, templates etc. the same way you would in a +# Google Cloud deployment configuration +resources: + resources: + # generates the bucket that the cloud function will send the backups to + # there is another bucket also created for uploading the package (sls-cloud-sql-backups-production-1603283101338), not to be confused with this one + - type: storage.v1.bucket + name: ${env:GOOGLE_PROJECT_ID}-sql-backups +# imports: +# - path: my_template.jinja diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py new file mode 100755 index 000000000..f58f86e98 --- /dev/null +++ b/gcp_service_account_credential_keys.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-10-29 18:02:14 +0000 (Thu, 29 Oct 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Lists all service account credential keys in a given GCP project + +Excludes built-in system managed keys which are hidden in the Console UI anyway and are not actionable +or in scope for a key policy audit. + + +Output Format: + + + + +You can supply a service account credentials file to authenticate with or just use ADC via: + + gcloud auth application-default login + + +See Also - similar scripts in the DevOps Bash tools repo: + + https://github.com/HariSekhon/DevOps-Bash-tools/ + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from datetime import datetime +import json +import os +import sys +import traceback +from google.oauth2 import service_account +import googleapiclient.discovery +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, log_option, validate_int + from harisekhon import CLI +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1' + + +class GcpServiceAccountCredentialKeys(CLI): + + def __init__(self): + # Python 2.x + super(GcpServiceAccountCredentialKeys, self).__init__() + # Python 3.x + # super().__init__() + self.credentials_file = None + self.service = None + self.project = None + self.no_expiry = None + self.expired = None + self.expires_within_days = None + + def add_options(self): + super(GcpServiceAccountCredentialKeys, self).add_options() + self.add_opt('-f', '--credentials-file', metavar='', + default=os.getenv('GOOGLE_CREDENTIALS', \ + os.getenv('GOOGLE_APPLICATION_CREDENTIALS')), + help='Credentials file ($GOOGLE_CREDENTIALS, ' + \ + '$GOOGLE_APPLICATION_CREDENTIALS)') + self.add_opt('-p', '--project-id', metavar='', + help='Google Cloud Project ID ($GOOGLE_PROJECT_ID, or inferred from credentials file') + self.add_opt('-n', '--no-expiry', action='store_true', help='List only non-expiring keys') + self.add_opt('-e', '--expired', action='store_true', help='List only expired keys') + self.add_opt('-d', '--expires-within-days', type=int, help='List only keys that will expire within N days') + + def process_options(self): + super(GcpServiceAccountCredentialKeys, self).process_options() + self.no_args() + project_id = self.get_opt('project_id') + credsfile = self.get_opt('credentials_file') + self.no_expiry = self.get_opt('no_expiry') + self.expired = self.get_opt('expired') + self.expires_within_days = self.get_opt('expires_within_days') + #if not credsfile: + # self.usage('no --credentials-file given and ' + \ + # 'GOOGLE_CREDENTIALS / GOOGLE_APPLICATION_CREDENTIALS environment variables not populated') + if credsfile: + if not os.path.exists(credsfile): + self.usage('credentials file not found: {}'.format(credsfile)) + self.credentials_file = credsfile + log_option('credentials file', self.credentials_file) + if not project_id: + if credsfile: + json_data = json.loads(open(credsfile).read()) + project_id = json_data['project_id'] + else: + self.usage('--project-id not specified and no credentials file given from which to infer') + self.project = project_id + log_option('project', self.project) + if self.expires_within_days is not None: + validate_int(self.expires_within_days, 'expires within days', 0) + if self.no_expiry: + self.usage('--expires-within-days and --no-expiry are mutually exclusive') + if self.expired: + self.usage('--expires-within-days and --expired are mutually exclusive') + if self.no_expiry and self.expired: + self.usage('--expired and --no-expiry are mutually exclusive') + + def run(self): + # defaults to looking for $GOOGLE_APPLICATION_CREDENTIALS or using Application Default Credentials + # from 'gcloud auth application-default login' => ~/.config/gcloud/application_default_credentials.json + credentials = None + if self.credentials_file: + log.debug('loading credentials') + credentials = service_account.Credentials.from_service_account_file( + filename=self.credentials_file, + scopes=['https://www.googleapis.com/auth/cloud-platform'] + ) + + # cache_discovery=False avoids: + # ImportError: file_cache is unavailable when using oauth2client >= 4.0.0 or google-auth + self.service = googleapiclient.discovery.build('iam', 'v1', credentials=credentials, cache_discovery=False) + + for service_account_email in self.get_service_accounts(): + self.list_keys(service_account_email) + + def get_service_accounts(self): + """ Returns a list of service account email addresses """ + + log.debug('getting service accounts') + service_accounts = self.service.projects()\ + .serviceAccounts()\ + .list(name='projects/{}'.format(self.project))\ + .execute() + for account in service_accounts['accounts']: + yield account['email'] + + def list_keys(self, service_account_email): + log.debug("getting keys for service account '%s'", service_account_email) + keys = self.service.projects()\ + .serviceAccounts()\ + .keys()\ + .list(name='projects/-/serviceAccounts/' + service_account_email).execute() + + for key in keys['keys']: + if key['keyType'] == 'SYSTEM_MANAGED': + continue + _id = key['name'].split('/')[-1] + created_date = key['validAfterTime'] + expiry_date = key['validBeforeTime'] + created_datetime = datetime.strptime(created_date, "%Y-%m-%dT%H:%M:%SZ") + expiry_datetime = datetime.strptime(expiry_date, "%Y-%m-%dT%H:%M:%SZ") + age_timedelta = datetime.utcnow() - created_datetime + age_days = int(age_timedelta.total_seconds() / 86400) + expired = False + if expiry_date == '9999-12-31T23:59:59Z': + expires_in_days = 'NEVER' + else: + expires_in_timedelta = expiry_datetime - datetime.utcnow() + expires_in_days = int(expires_in_timedelta.total_seconds() / 86400) + if expires_in_days < 1: + expired = True + if self.no_expiry and (expires_in_days != 'NEVER'): + continue + if self.expired and expired: + continue + if self.expires_within_days is not None and \ + (expires_in_days == 'NEVER' or expires_in_days > self.expires_within_days): + continue + print('{id} {created} {expires} {age:4d} {expires_in:5s} {expired} {service_account}'.format( + id=_id, + created=created_date, + expires=expiry_date, + age=age_days, + expires_in=expires_in_days, + expired=expired, + service_account=service_account_email + )) + + +if __name__ == '__main__': + GcpServiceAccountCredentialKeys().main() diff --git a/getent.py b/getent.py index ce77aefcf..d8b701cd6 100755 --- a/getent.py +++ b/getent.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-11-20 12:35:49 +0000 (Sun, 20 Nov 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/git_check_branches_upstream.py b/git_check_branches_upstream.py index 0ab2e177b..27ec09865 100755 --- a/git_check_branches_upstream.py +++ b/git_check_branches_upstream.py @@ -1,24 +1,24 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-07-21 16:19:19 +0100 (Thu, 21 Jul 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ Tool to check Git branches have their upstream origin branch set consistently and auto-fix if necessary -Mainly written for my https://github.com/harisekhon/Dockerfiles repo +Mainly written for my https://github.com/HariSekhon/Dockerfiles repo which has over 100 branches which get merged, pulled and pushed around """ diff --git a/hbase_compact_tables.py b/hbase_compact_tables.py index 60f246328..7bfca40b2 100755 --- a/hbase_compact_tables.py +++ b/hbase_compact_tables.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-04-27 20:49:23 +0100 (Wed, 27 Apr 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_flush_tables.py b/hbase_flush_tables.py index c4f43ef8d..71be25c17 100755 --- a/hbase_flush_tables.py +++ b/hbase_flush_tables.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-09-12 13:44:24 +0200 (Mon, 12 Sep 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # NOTE: 'flush' is not supported in Happybase as it's not in the Thrift API, @@ -44,7 +44,7 @@ import sys import traceback import subprocess -from subprocess import PIPE +PIPE = subprocess.PIPE libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) try: diff --git a/hbase_generate_data.py b/hbase_generate_data.py index 133fa42dc..3ed31a644 100755 --- a/hbase_generate_data.py +++ b/hbase_generate_data.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-09-14 15:19:35 +0200 (Wed, 14 Sep 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_region_requests.py b/hbase_region_requests.py index 2af24c219..d182431c8 100755 --- a/hbase_region_requests.py +++ b/hbase_region_requests.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_regions_by_memstore_size.py b/hbase_regions_by_memstore_size.py index 47165a7a9..295653a81 100755 --- a/hbase_regions_by_memstore_size.py +++ b/hbase_regions_by_memstore_size.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_regions_by_size.py b/hbase_regions_by_size.py index 49f7419d8..c67184c8b 100755 --- a/hbase_regions_by_size.py +++ b/hbase_regions_by_size.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_regions_least_used.py b/hbase_regions_least_used.py index 45727dd34..752be4ed5 100755 --- a/hbase_regions_least_used.py +++ b/hbase_regions_least_used.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_regionserver_requests.py b/hbase_regionserver_requests.py index f61b40cbd..a8c96ebb9 100755 --- a/hbase_regionserver_requests.py +++ b/hbase_regionserver_requests.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_scan_table_column_names.sh b/hbase_scan_table_column_names.sh index f2d62ad44..026f7d5a8 100755 --- a/hbase_scan_table_column_names.sh +++ b/hbase_scan_table_column_names.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2018-06-29 19:01:22 +0100 (Fri, 29 Jun 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/hbase_show_table_region_ranges.py b/hbase_show_table_region_ranges.py index 2a5647fb9..585e11cf9 100755 --- a/hbase_show_table_region_ranges.py +++ b/hbase_show_table_region_ranges.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-10-05 13:57:37 +0100 (Wed, 05 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_table_region_row_distribution.py b/hbase_table_region_row_distribution.py index 87b4d2e34..0dfe6e024 100755 --- a/hbase_table_region_row_distribution.py +++ b/hbase_table_region_row_distribution.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_table_regions_by_regionserver.sh b/hbase_table_regions_by_regionserver.sh index 5e603e289..8017f406a 100755 --- a/hbase_table_regions_by_regionserver.sh +++ b/hbase_table_regions_by_regionserver.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2018-08-24 19:34:33 +0100 (Fri, 24 Aug 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/hbase_table_regionserver_distribution.sh b/hbase_table_regionserver_distribution.sh index 252a9a3d4..58699a877 100755 --- a/hbase_table_regionserver_distribution.sh +++ b/hbase_table_regionserver_distribution.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2018-07-11 19:36:23 +0100 (Wed, 11 Jul 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Useful to see table region skew across regionservers diff --git a/hbase_table_row_key_distribution.py b/hbase_table_row_key_distribution.py index 8784d5952..6059763d7 100755 --- a/hbase_table_row_key_distribution.py +++ b/hbase_table_row_key_distribution.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-10-08 09:02:01 +0100 (Sat, 08 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hbase_uncompacted_regions.sh b/hbase_uncompacted_regions.sh index 7b092c06d..00a3302c5 100755 --- a/hbase_uncompacted_regions.sh +++ b/hbase_uncompacted_regions.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2018-07-11 19:26:21 +0100 (Wed, 11 Jul 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Find regions that require major_compact after region migration to achieve data locality again diff --git a/hdfs_files_native_checksums.jy b/hdfs_files_native_checksums.jy index 6aa6e6ff2..9c558401a 100755 --- a/hdfs_files_native_checksums.jy +++ b/hdfs_files_native_checksums.jy @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: 2013-06-20 18:21:02 +0100 (Thu, 20 Jun 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # @@ -50,7 +50,7 @@ def usage(*msg): if msg: printerr("".join(msg)) die(""" -Hari Sekhon - https://github.com/harisekhon/devops-python-tools +Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools ================================================================================ %s - version %s diff --git a/hdfs_files_stats.jy b/hdfs_files_stats.jy index 69132daee..587b3b390 100755 --- a/hdfs_files_stats.jy +++ b/hdfs_files_stats.jy @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: 2013-06-08 22:06:27 +0100 (Sat, 08 Jun 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # @@ -55,7 +55,7 @@ def usage(*msg): if msg: printerr(msg) die(""" -Hari Sekhon - https://github.com/harisekhon/devops-python-tools +Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools ================================================================================ %s - version %s diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py index 0a38a9316..1f348556d 100755 --- a/hdfs_find_replication_factor_1.py +++ b/hdfs_find_replication_factor_1.py @@ -1,18 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2018-11-28 16:37:00 +0000 (Wed, 28 Nov 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hdfs_time_block_reads.jy b/hdfs_time_block_reads.jy index e340cff7c..ab1812a23 100755 --- a/hdfs_time_block_reads.jy +++ b/hdfs_time_block_reads.jy @@ -3,7 +3,7 @@ # Author: Hari Sekhon # Date: 2013-06-08 22:06:27 +0100 (Sat, 08 Jun 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # @@ -28,7 +28,7 @@ __version__ = '0.9.1' import os usage_msg = """ -Hari Sekhon - https://github.com/harisekhon/devops-python-tools +Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools ================================================================================ %s - version %s diff --git a/headtail.py b/headtail.py index ff061445a..ec21a3f5c 100755 --- a/headtail.py +++ b/headtail.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-01-07 22:57:18 +0000 (Thu, 07 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # pylint: disable=line-too-long # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -41,7 +41,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.3.1' +__version__ = '0.3.2' class HeadTail(CLI): @@ -60,6 +60,7 @@ def __init__(self): self.sep = '-' * 80 self.docsep = '=' * 80 self.quiet = False + self.timeout_default = None def add_options(self): #self.timeout_default = 300 diff --git a/hexanonymize.py b/hexanonymize.py index 91728e7fc..cac35e6a0 100755 --- a/hexanonymize.py +++ b/hexanonymize.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2020-01-02 17:08:32 +0000 (Thu, 02 Jan 2020) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py new file mode 100755 index 000000000..d416f9379 --- /dev/null +++ b/hive_compute_table_stats.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to a HiveServer2 and compute optimization statistics on all tables, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Hive 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +# pylint: disable=import-error +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'HIVESERVER2_HOST', + 'HIVE_HOST', + 'HOST' +] + +port_envs = [ + 'HIVESERVER2_PORT', + 'HIVE_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Computes statistics on all Hive tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='HiveServer2 host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 10000), + help='HiveServer2 port (default: 10000, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, None) + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + compute_table_stats(conn, args, database, table, partition_regex) + +def compute_table_stats(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + partitions_found = False + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {}'.format(database)) + partition_cursor.execute('show partitions {}'.format(database)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + partitions_found = True + if not partition_regex.match(partition_value): + # pylint: disable=logging-not-lazy + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('ANALYZE TABLE {db}.{table} PARTITION({key}={value}) COMPUTE STATISTICS'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + if not partitions_found: + log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) + with conn.cursor() as table_cursor: + log.info("running compute stats on table '%s'", table) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute('ANALYZE TABLE {db}.{table} COMPUTE STATISTICS'.format(db=database, table=table)) + + +if __name__ == '__main__': + main() diff --git a/hive_foreach_table.py b/hive_foreach_table.py index 4ef224290..9d5abf6d6 100755 --- a/hive_foreach_table.py +++ b/hive_foreach_table.py @@ -1,22 +1,22 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ -Connect to HiveServer2 and execute a query for all tables in all databases, +Connect to HiveServer2 and execute a query for each table in each database, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py index 5daf009d6..ea41f3753 100755 --- a/hive_schemas_csv.py +++ b/hive_schemas_csv.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py new file mode 100755 index 000000000..d6cb4ce68 --- /dev/null +++ b/hive_tables_column_counts.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to HiveServer2 and count the number of columns for each table in each database, +or only those matching given db / table regexes + +You can also get this from the schemas.csv output generated by hive_schemas_csv.py, eg. + + tail -n +2 hive_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveTablesColumnCounts(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # not needed, here to suppress --query CLI option + self.database = None + self.table = None + self.ignore_errors = False + + # discarding last param query + def execute(self, conn, database, table, query): + column_count = 0 + log.info("describing table '%s.%s'", database, table) + with conn.cursor() as column_cursor: + # doesn't support parameterized query quoting from dbapi spec + #column_cursor.execute('use %(database)s', {'database': database}) + #column_cursor.execute('describe %(table)s', {'table': table}) + column_cursor.execute('use `{}`'.format(database)) + # don't use desc here, Impala doesn't support it and would break subclass + column_cursor.execute('describe `{}`'.format(table)) + for _ in column_cursor: + #column = _[0] + #column_type = _[1] + column_count += 1 + print('{db}.{table}\t{column_count}'.format(db=database, table=table, column_count=column_count)) + + +if __name__ == '__main__': + HiveTablesColumnCounts().main() diff --git a/hive_tables_list.py b/hive_tables_list.py index b0c4f4702..64675c61f 100755 --- a/hive_tables_list.py +++ b/hive_tables_list.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -63,6 +63,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.1.0' + class HiveTablesList(HiveForEachTable): def __init__(self): diff --git a/hive_tables_locations.py b/hive_tables_locations.py new file mode 100755 index 000000000..403e07443 --- /dev/null +++ b/hive_tables_locations.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to HiveServer2 and list the locations of all tables in all databases, +or only those matching given db / table regexes + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_metadata import HiveTablesMetadata +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveTablesLocations(HiveTablesMetadata): + + def __init__(self): + # Python 2.x + super(HiveTablesLocations, self).__init__() + # Python 3.x + # super().__init__() + self.field = 'Location' + + +if __name__ == '__main__': + HiveTablesLocations().main() diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py new file mode 100755 index 000000000..d15d62feb --- /dev/null +++ b/hive_tables_metadata.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to HiveServer2 and print the first matching DDL metadata field (eg. 'Location') +for each table in each database, or only those matching given db / table regexes + +Examples (fields are case sensitive regex and return N/A without match): + +./hive_tables_metadata.py --field Location ... +./hive_tables_metadata.py --field SerDe ... + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import re +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_regex + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.5.1' + + +class HiveTablesMetadata(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesMetadata, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'describe formatted {table}' + self.field = None + + def add_options(self): + # Python 2.x + super(HiveTablesMetadata, self).add_options() + # Python 3.x + # super().__init__() + if self.field is None: + self.add_opt('-f', '--field', help='Table DDL metadata field to return for each table (required)') + + def process_options(self): + # Python 2.x + super(HiveTablesMetadata, self).process_options() + # Python 3.x + # super().__init__() + if self.field is None: + self.field = self.get_opt('field') + if not self.field: + self.usage('--field not specified') + validate_regex(self.field, 'field') + self.field = re.compile(self.field) + + # discard last param query and construct our own based on the table DDL of cols + def execute(self, conn, database, table, query): + log.info("describing table '%s.%s'", database, table) + field = 'N/A' + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + #table_cursor.execute('describe %(table)s', {'table': table}) + table_cursor.execute('use `{}`'.format(database)) + table_cursor.execute(query.format(table=table)) + for row in table_cursor: + if self.field.search(row[0]): + field = row[1] + break + print('{db}.{table}\t{field}'.format(db=database, table=table, field=field)) + + +if __name__ == '__main__': + HiveTablesMetadata().main() diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py index a78e049f4..207fcba3f 100755 --- a/hive_tables_null_columns.py +++ b/hive_tables_null_columns.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -67,6 +67,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class HiveTablesNullColumns(HiveForEachTable): def __init__(self): @@ -81,7 +82,7 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): + def execute(self, conn, database, table, _query): sum_part = '' columns = [] log.info("describing table '%s.%s'", database, table) diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py index 78c42d15e..f1a5752f1 100755 --- a/hive_tables_null_rows.py +++ b/hive_tables_null_rows.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -62,7 +62,8 @@ __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.6.0' + class HiveTablesNullRows(HiveForEachTable): @@ -78,11 +79,11 @@ def __init__(self): self.ignore_errors = False # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): + def execute(self, conn, database, table, _): columns = [] log.info("describing table '%s.%s'", database, table) with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec + # impala library doesn't support parameterized query quoting from dbapi spec #column_cursor.execute('use %(database)s', {'database': database}) #column_cursor.execute('describe %(table)s', {'table': table}) column_cursor.execute('use `{}`'.format(database)) @@ -91,9 +92,7 @@ def execute(self, conn, database, table, query): column = column_row[0] #column_type = column_row[1] columns.append(column) - query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ - .format(db=database, table=table) + \ - "` IS NULL AND `".join(columns) + "` IS NULL" + query = self.generate_sql(database, table, columns) with conn.cursor() as table_cursor: log.debug('executing query: %s', query) table_cursor.execute(query) @@ -101,6 +100,13 @@ def execute(self, conn, database, table, query): count = result[0] print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) + @staticmethod + def generate_sql(database, table, columns): + sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL AND `".join(columns) + "` IS NULL" + return sql + if __name__ == '__main__': HiveTablesNullRows().main() diff --git a/hive_tables_row_column_counts.py b/hive_tables_row_column_counts.py new file mode 100755 index 000000000..c2a5ec7e9 --- /dev/null +++ b/hive_tables_row_column_counts.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to HiveServer2 and count the number of rows and columns for each table +in each database, or only those matching given db / table regexes + +Output format: + + . + +Tested on Hive 1.1.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log + from hive_foreach_table import HiveForEachTable +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + + +__author__ = 'Hari Sekhon' +__version__ = '0.5.0' + + +class HiveTablesRowColumnCounts(HiveForEachTable): + + def __init__(self): + # Python 2.x + super(HiveTablesRowColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + self.query = 'placeholder' # not needed, here to suppress --query CLI option + self.database = None + self.table = None + self.ignore_errors = False + + # discarding last param query + def execute(self, conn, database, table, query): + row_count = None + column_count = 0 + log.info("describing table '%s.%s'", database, table) + with conn.cursor() as cursor: + # doesn't support parameterized query quoting from dbapi spec + #cursor.execute('use %(database)s', {'database': database}) + #cursor.execute('describe %(table)s', {'table': table}) + cursor.execute('use `{}`'.format(database)) + # don't use desc here, Impala doesn't support it and would break subclass + cursor.execute('describe `{}`'.format(table)) + for _ in cursor: + #column = _[0] + #column_type = _[1] + column_count += 1 + log.info("running SELECT COUNT(*) FROM `%s`.`%s`", database, table) + # doesn't support parameterized query quoting from dbapi spec + cursor.execute('SELECT COUNT(*) FROM `{db}`.`{table}`'.format(db=database, table=table)) + for result in cursor: + assert row_count is None + row_count = result[0] + print('{db}.{table}\t{row_count}\t{column_count}'\ + .format(db=database, + table=table, + row_count=row_count, + column_count=column_count)) + + +if __name__ == '__main__': + HiveTablesRowColumnCounts().main() diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py index b37059c9b..27aaa1a9c 100755 --- a/hive_tables_row_counts.py +++ b/hive_tables_row_counts.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py index 0f7a5a94a..8ea7e6b75 100755 --- a/hive_tables_row_counts_any_nulls.py +++ b/hive_tables_row_counts_any_nulls.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -52,8 +52,7 @@ sys.path.append(pylib) try: # pylint: disable=wrong-import-position - from harisekhon.utils import log - from hive_foreach_table import HiveForEachTable + from hive_tables_null_rows import HiveTablesNullRows except ImportError as _: print('module import failed: %s' % _, file=sys.stderr) print("Did you remember to build the project by running 'make'?", file=sys.stderr) @@ -62,44 +61,18 @@ __author__ = 'Hari Sekhon' -__version__ = '0.5.0' - -class HiveTablesRowsWithNulls(HiveForEachTable): - - def __init__(self): - # Python 2.x - super(HiveTablesRowsWithNulls, self).__init__() - # Python 3.x - # super().__init__() - self.query = 'placeholder' # constructed later dynamically per table, here to suppress --query CLI option - self.database = None - self.table = None - #self.partition = None - self.ignore_errors = False - - # discard last param query and construct our own based on the table DDL of cols - def execute(self, conn, database, table, query): - columns = [] - log.info("describing table '%s.%s'", database, table) - with conn.cursor() as column_cursor: - # doesn't support parameterized query quoting from dbapi spec - #column_cursor.execute('use %(database)s', {'database': database}) - #column_cursor.execute('describe %(table)s', {'table': table}) - column_cursor.execute('use `{}`'.format(database)) - column_cursor.execute('describe `{}`'.format(table)) - for column_row in column_cursor: - column = column_row[0] - #column_type = column_row[1] - columns.append(column) - query = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ - .format(db=database, table=table) + \ - "` IS NULL OR `".join(columns) + "` IS NULL" - with conn.cursor() as table_cursor: - log.debug('executing query: %s', query) - table_cursor.execute(query) - for result in table_cursor: - count = result[0] - print('{db}.{table}\t{count}'.format(db=database, table=table, count=count)) +__version__ = '0.6.0' + + +class HiveTablesRowsWithNulls(HiveTablesNullRows): + + @staticmethod + def generate_sql(database, table, columns): + # impala library doesn't support parameterized query quoting from dbapi spec + sql = "SELECT count(*) FROM `{db}`.`{table}` WHERE `"\ + .format(db=database, table=table) + \ + "` IS NULL OR `".join(columns) + "` IS NULL" + return sql if __name__ == '__main__': diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py new file mode 100755 index 000000000..04d462e16 --- /dev/null +++ b/impala_compute_table_stats.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to a Impalad and compute optimization statistics on all tables, +or only those matching given db / table / partition value regexes + +Tested on CDH 5.10, Impala 1.1.0 with Kerberos + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import logging +import os +import re +import socket +import sys +# pylint: disable=import-error +from impala.dbapi import connect + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + +logging.basicConfig() +log = logging.getLogger(os.path.basename(sys.argv[0])) + +host_envs = [ + 'IMPALA_HOST', + 'HOST' +] + +port_envs = [ + 'IMPALA_PORT', + 'PORT' +] + +def getenvs(keys, default=None): + for key in keys: + value = os.getenv(key) + if value: + return value + return default + +def parse_args(): + parser = argparse.ArgumentParser( + description="Computes statistics on all Impala tables/partitions matching database / table / partition regexes") + parser.add_argument('-H', '--host', default=getenvs(host_envs, socket.getfqdn()),\ + help='Impalad host ' + \ + '(default: fqdn of local host, $' + ', $'.join(host_envs) + ')') + parser.add_argument('-P', '--port', type=int, default=getenvs(port_envs, 21050), + help='Impalad port (default: 21050, ' + ', $'.join(port_envs) + ')') + parser.add_argument('-d', '--database', default='.*', help='Database regex (default: .*)') + parser.add_argument('-t', '--table', default='.*', help='Table regex (default: .*)') + parser.add_argument('-p', '--partition', default='.*', help='Partition regex (default: .*)') + parser.add_argument('-k', '--kerberos', action='store_true', help='Use Kerberos (you must kinit first)') + parser.add_argument('-n', '--krb5-service-name', default='hive', + help='Service principal (default: \'hive\')') + parser.add_argument('-S', '--ssl', action='store_true', help='Use SSL') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.INFO) + + return args + +def connect_db(args, database): + auth_mechanism = None + if args.kerberos: + auth_mechanism = 'GSSAPI' + + log.info('connecting to %s:%s database %s', args.host, args.port, database) + return connect( + host=args.host, + port=args.port, + auth_mechanism=auth_mechanism, + use_ssl=args.ssl, + #user=user, + #password=password, + database=database, + kerberos_service_name=args.krb5_service_name + ) + +def main(): + args = parse_args() + + try: + database_regex = re.compile(args.database, re.I) + table_regex = re.compile(args.table, re.I) + partition_regex = re.compile(args.partition, re.I) + except re.error as _: + log.error('error in provided regex: %s', _) + sys.exit(3) + + conn = connect_db(args, None) + + log.info('querying databases') + with conn.cursor() as db_cursor: + db_cursor.execute('show databases') + for db_row in db_cursor: + database = db_row[0] + if not database_regex.search(database): + log.debug("skipping database '%s', does not match regex '%s'", database, args.database) + continue + log.info('querying tables for database %s', database) + #db_conn = connect_db(args, database) + #with db_conn.cursor() as table_cursor: + with conn.cursor() as table_cursor: + # doesn't support parameterized query quoting from dbapi spec + #table_cursor.execute('use %(database)s', {'database': database}) + table_cursor.execute('use {}'.format(database)) + table_cursor.execute('show tables') + for table_row in table_cursor: + table = table_row[0] + if not table_regex.search(table): + log.debug("skipping database '%s' table '%s', does not match regex '%s'", \ + database, table, args.table) + continue + compute_table_stats(conn, args, database, table, partition_regex) + +def compute_table_stats(conn, args, database, table, partition_regex): + log.info("getting partitions for database '%s' table '%s'", database, table) + partitions_found = False + with conn.cursor() as partition_cursor: + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('use {}'.format(database)) + partition_cursor.execute('show partitions {}'.format(database)) + for partitions_row in partition_cursor: + partition_key = partitions_row[0] + partition_value = partitions_row[1] + partitions_found = True + if not partition_regex.match(partition_value): + # pylint: disable=logging-not-lazy + log.debug("skipping database '%s' table '%s' partition key '%s' value '%s', " + + "value does not match regex '%s'", + database, + table, + partition_key, + partition_value, + args.partition) + continue + # doesn't support parameterized query quoting from dbapi spec + partition_cursor.execute('COMPUTE INCREMENTAL STATS {db}.{table} PARTITION({key}={value})'\ + .format(db=database, table=table, key=partition_key, value=partition_value)) + if not partitions_found: + log.info("no partitions found for database '%s' table '%s', computing stats for whole table", database, table) + with conn.cursor() as table_cursor: + log.info("running compute stats on table '%s'", table) + # doesn't support parameterized query quoting from dbapi spec + table_cursor.execute('COMPUTE STATS {db}.{table}'.format(db=database, table=table)) + + +if __name__ == '__main__': + main() diff --git a/impala_foreach_table.py b/impala_foreach_table.py index 7b35b0174..f4e324ac9 100755 --- a/impala_foreach_table.py +++ b/impala_foreach_table.py @@ -1,22 +1,22 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ -Connect to an Impala daemon and execute a query for all tables in all databases, +Connect to an Impala daemon and execute a query for each table in each database, or only those matching given db / table regexes Useful for getting row counts of all tables or analyzing tables: diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py index d6555b542..0ac8491dd 100755 --- a/impala_schemas_csv.py +++ b/impala_schemas_csv.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py new file mode 100755 index 000000000..e62027542 --- /dev/null +++ b/impala_tables_column_counts.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to an Impala daemon and count the number of columns for each table in each database, +or only those matching given db / table regexes + +You can also get this from the schemas.csv output generated by impala_schemas_csv.py, eg. + + tail -n +2 impala_schemas.csv | cut -d, -f1,2 | sed 's/"//g; s/,/./' | sort | uniq -c | sort -k1nr + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_column_counts import HiveTablesColumnCounts +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesColumnCounts(HiveTablesColumnCounts): + + def __init__(self): + # Python 2.x + super(ImpalaTablesColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesColumnCounts().main() diff --git a/impala_tables_list.py b/impala_tables_list.py index feb4e73ca..a45194ad3 100755 --- a/impala_tables_list.py +++ b/impala_tables_list.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/impala_tables_locations.py b/impala_tables_locations.py new file mode 100755 index 000000000..383bb1154 --- /dev/null +++ b/impala_tables_locations.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to an Impala daemon and list the locations of all tables in all databases, +or only those matching given db / table regexes + +Caveats: + + Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't + + Impala is faster than Hive for the first ~1000 tables but then slows down + so if you have a lot of tables I recommend you use the Hive version of this instead + eg. by ~1900 tables the Hive version will overtake the Impala version and + for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_locations import HiveTablesLocations +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesLocations(HiveTablesLocations): + + def __init__(self): + # Python 2.x + super(ImpalaTablesLocations, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesLocations().main() diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py new file mode 100755 index 000000000..501b82c93 --- /dev/null +++ b/impala_tables_metadata.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to an Impala daemon and print the first matching DDL metadata field (eg. 'Location') +for each table in each database, or only those matching given db / table regexes + +Examples (fields are case sensitive regex and return N/A without match): + +./impala_tables_metadata.py --field Location ... +./impala_tables_metadata.py --field SerDe ... + +Caveats: + + Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't + + Impala is faster than Hive for the first ~1000 tables but then slows down + so if you have a lot of tables I recommend you use the Hive version of this instead + eg. by ~1900 tables the Hive version will overtake the Impala version and + for thousands of tables Impala actuallys runs 1.5 - 2x slower than the Hive version overall + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_metadata import HiveTablesMetadata +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesMetadata(HiveTablesMetadata): + + def __init__(self): + # Python 2.x + super(ImpalaTablesMetadata, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesMetadata().main() diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py index ad968fbc1..2b70caf3f 100755 --- a/impala_tables_null_columns.py +++ b/impala_tables_null_columns.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py index c70484540..4fea8a82e 100755 --- a/impala_tables_null_rows.py +++ b/impala_tables_null_rows.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/impala_tables_row_column_counts.py b/impala_tables_row_column_counts.py new file mode 100755 index 000000000..8cf4d6b82 --- /dev/null +++ b/impala_tables_row_column_counts.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Connect to an Impala daemon and count the number of rows and columns for each table +in each database, or only those matching given db / table regexes + +Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL + +Due to a thrift / impyla bug this needs exactly thrift==0.9.3, see + +https://github.com/cloudera/impyla/issues/286 + +If you get an error like this: + +ERROR:impala.hiveserver2:Failed to open transport (tries_left=1) +... +TTransportException: TSocket read 0 bytes + +then check your --kerberos and --ssl settings match the cluster's settings +(Thrift and Kerberos have the worst error messages ever) + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys +srcdir = os.path.abspath(os.path.dirname(__file__)) +pylib = os.path.join(srcdir, 'pylib') +sys.path.append(pylib) +try: + # pylint: disable=wrong-import-position + from hive_tables_row_column_counts import HiveTablesRowColumnCounts +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.4.0' + + +class ImpalaTablesRowColumnCounts(HiveTablesRowColumnCounts): + + def __init__(self): + # Python 2.x + super(ImpalaTablesRowColumnCounts, self).__init__() + # Python 3.x + # super().__init__() + + # these are auto-set checking sys.argv[0] in HiveImpalaCLI class + self.name = 'Impala' + #self.default_port = 21050 + #self.default_service_name = 'impala' + + +if __name__ == '__main__': + ImpalaTablesRowColumnCounts().main() diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py index 47c104789..decc2982c 100755 --- a/impala_tables_row_counts.py +++ b/impala_tables_row_counts.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -62,6 +62,7 @@ __author__ = 'Hari Sekhon' __version__ = '0.5.0' + class ImpalaTablesRowCounts(HiveTablesRowCounts): def __init__(self): diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py index ab6a6c638..d7b49a3b2 100755 --- a/impala_tables_row_counts_any_nulls.py +++ b/impala_tables_row_counts_any_nulls.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/ipython_notebook_pyspark.py b/ipython_notebook_pyspark.py index 013d61ff2..4c4505015 100755 --- a/ipython_notebook_pyspark.py +++ b/ipython_notebook_pyspark.py @@ -1,9 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Author: Hari Sekhon # Date: 6/8/2014 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # @@ -93,7 +93,7 @@ # TODO: rewrite from here as a CLI class, not using globals if len(sys.argv) > 1: - printerr("""Hari Sekhon - https://github.com/harisekhon/devops-python-tools + printerr("""Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools usage: %s diff --git a/json_docs_to_bulk_multiline.py b/json_docs_to_bulk_multiline.py index 98f4d4db9..eb49bcbeb 100755 --- a/json_docs_to_bulk_multiline.py +++ b/json_docs_to_bulk_multiline.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2017-07-28 17:08:47 +0200 (Fri, 28 Jul 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/json_to_xml.py b/json_to_xml.py index 03f5b27fa..54c0d6395 100755 --- a/json_to_xml.py +++ b/json_to_xml.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-01-15 00:07:09 +0000 (Fri, 15 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/json_to_yaml.py b/json_to_yaml.py index b3c2b4501..674f6729f 100755 --- a/json_to_yaml.py +++ b/json_to_yaml.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-12-19 17:54:21 +0000 (Thu, 19 Dec 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -29,7 +29,8 @@ See also: - https://www.commandlinefu.com/commands/view/12221/convert-json-to-yaml + + json2yaml.sh - https://github.com/HariSekhon/DevOps-Bash-tools """ diff --git a/jython_autoinstall.exp b/jython_autoinstall.exp index 1230a03a1..342488e6a 100755 --- a/jython_autoinstall.exp +++ b/jython_autoinstall.exp @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: Fri Jun 17 15:12:17 2016 +0100 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set force_conservative 0 ;# set to 1 to force conservative mode even if diff --git a/jython_install.sh b/jython_install.sh index b56c94cac..4d3f420ee 100755 --- a/jython_install.sh +++ b/jython_install.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-08-01 10:17:55 +0100 (Mon, 01 Aug 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/kics.config b/kics.config new file mode 100644 index 000000000..51ac5712e --- /dev/null +++ b/kics.config @@ -0,0 +1,44 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-05 18:05:53 +0100 (Fri, 05 May 2023) +# +# vim:ts=2:sts=2:sw=2:et:filetype=yaml +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# K i c s C o n f i g +# ============================================================================ # + +# https://github.com/Checkmarx/kics/blob/master/docs/configuration-file.md + +--- +#path: assets/iac_samples +verbose: true +log-file: true +#type: +# - Dockerfile +# - Kubernetes +#queries-path: "assets/queries" +exclude-paths: + # ignore submodules - handle them in the source repos only + - bash-tools/ + - github-actions/ + - haproxy-configs/ + - jenkins/ + - kubernetes-templates/ + - lib/ + - pylib/ + - spotify-tools/ + - sql/ + - sql-keywords/ + - templates/ + - terraform-templates/ +#output-path: "results" diff --git a/lib/hive_impala_cli.py b/lib/hive_impala_cli.py index 7515abddd..958ccec69 100755 --- a/lib/hive_impala_cli.py +++ b/lib/hive_impala_cli.py @@ -4,14 +4,14 @@ # Author: Hari Sekhon # Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # from __future__ import absolute_import diff --git a/lib/postgres_cli.py b/lib/postgres_cli.py new file mode 100755 index 000000000..e84416d98 --- /dev/null +++ b/lib/postgres_cli.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-12 17:39:57 +0000 (Thu, 12 Mar 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import socket +import sys +import psycopg2 +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import log, validate_host, validate_port, validate_user, validate_password + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.1.0' + + +class PostgreSQLCLI(CLI): + + def __init__(self): + # Python 2.x + super(PostgreSQLCLI, self).__init__() + # Python 3.x + # super().__init__() + self.name = ['PostgreSQL', 'Postgres', 'PG'] + self.host = None + self.port = None + self.default_host = socket.getfqdn() + self.default_port = 5432 + self.user = None + self.password = None + #self.ssl = False + self.verbose_default = 1 + self.timeout_default = None + + def add_options(self): + super(PostgreSQLCLI, self).add_options() + self.add_hostoption() + self.add_useroption() + self.add_opt('-d', '--database', help='Database to connect to') + + def process_options(self): + super(PostgreSQLCLI, self).process_options() + self.host = self.get_opt('host') + self.host = self.get_opt('host') + self.user = self.get_opt('user') + self.password = self.get_opt('password') + validate_host(self.host) + validate_port(self.port) + validate_user(self.user) + validate_password(self.password) + self.port = int(self.port) + #self.ssl = self.get_opt('ssl') + + def connect(self, database): + log.info('connecting to %s:%s database %s as user %s', self.host, self.port, database) + return psycopg2.connect(host=self.host, + port=self.port, + database=database, + user=self.user, + password=self.password) diff --git a/opentsdb_import_metric_distribution.py b/opentsdb_import_metric_distribution.py index 1331b68d2..3b3d3df09 100755 --- a/opentsdb_import_metric_distribution.py +++ b/opentsdb_import_metric_distribution.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-10-10 11:47:12 +0100 (Mon, 10 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/opentsdb_list_metrics.sh b/opentsdb_list_metrics.sh index 6365ffc35..a119bd9fa 100755 --- a/opentsdb_list_metrics.sh +++ b/opentsdb_list_metrics.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2018-07-13 22:36:14 +0100 (Fri, 13 Jul 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/opentsdb_list_metrics_hbase.sh b/opentsdb_list_metrics_hbase.sh index 945b108ff..86142e4c0 100755 --- a/opentsdb_list_metrics_hbase.sh +++ b/opentsdb_list_metrics_hbase.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2018-07-13 22:36:14 +0100 (Fri, 13 Jul 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -91,6 +91,7 @@ print_metrics(){ print_metrics_by_age(){ # hackish but convenient - forking to the date command thousands or hundreds of thousands of times can take hours, python takes 10 secs even for 250,000+ metrics tmp_python_script=$(mktemp) + # shellcheck disable=SC1117 cat > "$tmp_python_script" <=2.0.0,<3.0.0 # for Elasticsearch 1.x #elasticsearch>=1.0.0,<2.0.0 + # fails on requiring newer version of setuptools #Flask==0.10.1 -GitPython==2.1.14 -happybase==1.0.0 +GitPython==2.1.15 + +# this GCP API is surprisingly awful, not using +#google-api-python-client==1.11.0 + +# XXX: install broken on new M1 Mac Python 3.9 +#happybase==1.0.0 + humanize==0.5.1 -impyla==0.16.0 -Jinja2==2.10.1 +impyla==0.19.0 +jinja2==2.11.3 #kazoo==2.2.1 ldif3==3.2.2 #MarkupSafe==0.23 #Markdown==2.6.8 + +matplotlib==3.7.5 + # Python 3.5+ #numpy==1.17.2 -numpy==1.16.5 +# XXX: install broken on new M1 Mac Python 3.9 +#numpy==1.16.5 + +# for plot_uk_marriage_rates.py +openpyxl==3.1.5 +pandas==2.0.3 + +# requires pg_config to build from source +#psycopg2==2.8.4 +# XXX: install broken on new M1 Mac Python 3.9 +#psycopg2-binary==2.8.4 + python-cson==1.0.9 -psutil==4.3.0 +psutil==5.7.0 + # parquet support in pyarrow is weaker, gone back to using parquet-tools #pyarrow==0.6.0 #PyHive==0.6.1 + +# doesn't work with non-trivial code #PyInstaller==3.3.1 -python-ldap==3.2.0 + +# gcc compile error on Alpine, don't think this is used either +#python-ldap==3.2.0 + #python-jenkins==0.4.13 # pulled in automatically by snakebite[kerberos] #python-krbV==1.0.90 # needed by avro -python-snappy==0.5 -sasl==0.2.1 + +# XXX: install broken on new M1 Mac Python 3.9 +#python-snappy==0.5 + +# XXX: install broken on new M1 Mac Python 3.9 +#sasl==0.2.1 + sh==1.12.14 +selenium==3.141.0 + # pulls in python-KrbV as a dependency which doesn't build on Mac any more -# moved to Makefile as best effort +# relies on python-krbV is unmaintained and unported to Python 3 +# - moved to Makefile as best effort #snakebite[kerberos]==2.11.0 -snakebite==2.11.0 -thrift-sasl==0.2.1 -thrift==0.9.3 -thriftpy==0.3.9 +#snakebite==2.11.0 + +# XXX: install broken on new M1 Mac Python 3.9 +#thrift-sasl==0.2.1 +#thrift==0.9.3 +#thriftpy==0.3.9 + toml==0.10.0 xmltodict==0.10.2 yamllint==1.15.0 + +#pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability. update: this breaks Python 3.5 build where this requirement is not found diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py new file mode 100755 index 000000000..c3d117e53 --- /dev/null +++ b/selenium_hub_browser_test.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-05-12 09:55:01 +0100 (Wed, 12 May 2021) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn +# and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Tests a Selenium Hub / Selenoid using the given browsers eg. FIREFOX, CHROME +against a given URL and content (defaults to google.com) + +Browsers default to 'FIREFOX' and 'CHROME' if not specified +URL defaults to 'google.com' checking for content 'google' +If you define a different URL then you must specify a --content or --regex validation otherwise none is used + +Example: + + ./selenium_hub_browser_test.py --host [] [] + + ./selenium_hub_browser_test.py --hub-url http://:4444/wd/hub/ [] [] + +Where browsers are one or more of these and must be supported by the remote Selenium Hub: + +ANDROID +CHROME +EDGE +FIREFOX +HTMLUNIT +HTMLUNITWITHJS +INTERNETEXPLORER +IPAD +IPHONE +OPERA +PHANTOMJS +SAFARI +WEBKITGTK + +Examples: + + ./selenium_hub_browser_test.py --host x.x.x.x + + ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME + + ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --content google + ./selenium_hub_browser_test.py --host x.x.x.x FIREFOX CHROME --url google.com --regex 'goog.*' + + +Tested on Selenium Grid Hub v.3.141.59, v4.0.0 and Selenoid 1.10.1 +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import re +import sys +import time +import traceback +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +srcdir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(srcdir, 'pylib') +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon import CLI + from harisekhon.utils import log, validate_host, validate_port, validate_url, validate_regex, die +except ImportError as _: + print(traceback.format_exc(), end='') + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.3' + + +class SeleniumHubBrowserTest(CLI): + + def __init__(self): + # Python 2.x + super(SeleniumHubBrowserTest, self).__init__() + # Python 3.x + # super().__init__() + self.host = None + self.port = None + self.protocol = 'http' + self.name = 'Selenium Hub' + self.path = 'wd/hub' + self.hub_url = None + self.url_default = 'http://google.com' + self.url = self.url_default + self.expected_content = None + self.expected_content_default = 'google' + self.expected_regex = None + self.timeout_default = 600 + self.verbose_default = 2 + + def add_options(self): + super(SeleniumHubBrowserTest, self).add_options() + self.add_hostoption(name='Selenium Hub', default_port=4444) + self.add_opt('-U', '--hub-url', help='Selenium Hub URL (takes priority over --host/--port/--ssl)') + self.add_opt('-u', '--url', default=self.url_default, + help='URL to use for the test (default: {})'.format(self.url_default)) + self.add_opt('-c', '--content', help='URL content to expect') + self.add_opt('-r', '--regex', help='URL content to expect') + self.add_opt('-S', '--ssl', action='store_true', help='Use SSL to connect to Selenium Hub') + + def process_options(self): + super(SeleniumHubBrowserTest, self).process_options() + self.hub_url = self.get_opt('hub_url') + if self.hub_url: + validate_url(self.hub_url, 'hub') + else: + self.host = self.get_opt('host') + self.port = self.get_opt('port') + validate_host(self.host) + validate_port(self.port) + if self.get_opt('ssl') or int(self.port) == 443: + self.protocol = 'https' + self.hub_url = '{protocol}://{host}:{port}/{path}'\ + .format(protocol=self.protocol, \ + host=self.host, \ + port=self.port, \ + path=self.path) + self.url = self.get_opt('url') + if ':' not in self.url: + self.url = 'http://' + self.url + validate_url(self.url) + self.expected_content = self.get_opt('content') + self.expected_regex = self.get_opt('regex') + if self.expected_regex: + validate_regex(self.expected_regex) + self.expected_regex = re.compile(self.expected_regex) + elif self.url == self.url_default: + self.expected_content = self.expected_content_default + if not self.args: + # test basic Chrome and Firefox are available + self.args.append('chrome') + self.args.append('firefox') + + def check_selenium(self, browser): + log.info("Connecting to '%s' for browser '%s'", self.hub_url, browser) + driver = webdriver.Remote( + command_executor=self.hub_url, + desired_capabilities=getattr(DesiredCapabilities, browser) + ) + log.info("Checking url '%s'", self.url) + driver.get(self.url) + content = driver.page_source + title = driver.title + driver.quit() + if self.expected_regex: + log.info("Checking url content matches regex") + if not self.expected_regex.search(content): + die('ERROR: Page source content failed regex search') + elif self.expected_content: + log.info("Checking url content matches '%s'", self.expected_content) + if self.expected_content not in content: + die('ERROR: Page source content failed content match') + # not really recommended but in this case we cannot predict + # what to expect on a random url if not specified by --content/--regex (provided in the default test case) + # + # https://www.selenium.dev/documentation/en/worst_practices/http_response_codes/ + elif '404' in title: + die('ERROR: Page title contains a 404 / error ' + + '(if this is expected, specify --content / --regex to check instead): {}'.format(title)) + log.info("Succeeded for browser '%s' against url '%s'", browser, self.url) + + def run(self): + start_time = time.time() + for browser in self.args: + self.check_selenium(browser.upper()) + query_time = time.time() - start_time + log.info('Finished checks in {:.2f} secs'.format(query_time)) + + +if __name__ == '__main__': + SeleniumHubBrowserTest().main() diff --git a/serf_event_handler.py b/serf_event_handler.py index 38a0df1b4..c0d5cacbb 100755 --- a/serf_event_handler.py +++ b/serf_event_handler.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-01-16 15:44:16 +0000 (Sat, 16 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -61,7 +61,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.2.2' +__version__ = '0.2.3' class SerfEventHandler(CLI): @@ -95,7 +95,7 @@ def add_options(self): def enable_commands(self): if self.event in ['query', 'event']: - cmd = None + cmd = '' if self.event == 'query': cmd = self.query_name elif self.event == 'user': diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt index f8738775b..a8fe03ea8 100644 --- a/setup/apk-packages-dev.txt +++ b/setup/apk-packages-dev.txt @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # @@ -14,5 +14,8 @@ # ============================================================================ # openldap-dev +postgresql-dev snappy-dev -unzip + +# installed by bash-tools submodule now +#unzip diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt index 88bcf4c3a..1b9ee9d77 100644 --- a/setup/apk-packages-pip.txt +++ b/setup/apk-packages-pip.txt @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # @@ -16,6 +16,8 @@ py-dicttoxml py-psutil py-pyldap -py2-jinja2 -py2-numpy -#py-flask +py3-jinja2 +py3-numpy +#py3-flask +py3-pygit2 +py3-psycopg2 diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt index fc6956249..267f430e5 100644 --- a/setup/apk-packages.txt +++ b/setup/apk-packages.txt @@ -2,35 +2,38 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Alpine Package Requirements # ============================================================================ # -# full mktemp needed for json tests -# full split needed for anonymize_parallel.sh -coreutils - -grep - -# needed for tests/test_spark* and ambari_cancel_all_requests.sh -curl - # for htpasswd for docker registry authenticated checks apache2-utils -# for anonymize_parallel.sh -parallel - #which java || $(SUDO) apk add openjdk8-jre-base # Spark Java Py4J gets java linking error without this #if [ -f /lib/libc.musl-x86_64.so.1 ]; then [ -e /lib/ld-linux-x86-64.so.2 ] || ln -sv /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2; fi -zip +# ===================================== +# installed by bash-tools submodule now + +# full mktemp needed for json tests +# full split needed for anonymize_parallel.sh +#coreutils + +# needed for tests/test_spark* and ambari_cancel_all_requests.sh +#curl + +#grep + +# for anonymize_parallel.sh +#parallel + +#zip diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh new file mode 100755 index 000000000..d7bedf92a --- /dev/null +++ b/setup/bootstrap.sh @@ -0,0 +1,86 @@ +#!/bin/sh +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2019-10-16 10:33:03 +0100 (Wed, 16 Oct 2019) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# Alpine / Wget: +# +# wget -O- https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh +# +# Curl: +# +# curl https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh + +set -eu +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(dirname "$0")" + +repo="https://github.com/HariSekhon/DevOps-Python-tools" + +directory="pytools" + +sudo="" +[ "$(whoami)" = "root" ] || sudo=sudo + +if [ "$(uname -s)" = Darwin ]; then + echo "Bootstrapping on Mac OS X: $repo" + if ! type brew >/dev/null 2>&1; then + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install | $sudo ruby + fi +elif [ "$(uname -s)" = Linux ]; then + echo "Bootstrapping on Linux: $repo" + if type apk >/dev/null 2>&1; then + $sudo apk --no-cache add bash git make curl wget + elif type apt-get >/dev/null 2>&1; then + if [ -n "${CI:-}" ]; then + export DEBIAN_FRONTEND=noninteractive + fi + opts="-o DPkg::Lock::Timeout=1200" + if [ -z "${PS1:-}" ]; then + opts="$opts -qq" + fi + $sudo apt-get update $opts + $sudo apt-get install $opts -y git make curl wget --no-install-recommends + elif type yum >/dev/null 2>&1; then + if grep -qi 'NAME=.*CentOS' /etc/*release; then + echo "CentOS EOL detected, replacing yum base URL to vault to re-enable package installs" + $sudo sed -i 's/^[[:space:]]*mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Linux-* + $sudo sed -i 's|^#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|' /etc/yum.repos.d/CentOS-Linux-* + fi + $sudo yum install -y git make curl wget + else + echo "Package Manager not found on Linux, cannot bootstrap" + exit 1 + fi +else + echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time" + exit 1 +fi + +if [ "${srcdir##*/}" = setup ]; then + cd "$srcdir/.." +else + # if this is an empty directory eg. a cache mount, then remove it to get a proper checkout + rmdir "$directory" 2>/dev/null || : + if [ -d "$directory" ]; then + cd "$directory" + git pull + else + git clone "$repo" "$directory" + cd "$directory" + fi +fi + +if [ -z "${NO_MAKE:-}" ]; then + make +fi diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt index fa25317ba..401e24b97 100644 --- a/setup/brew-packages.txt +++ b/setup/brew-packages.txt @@ -3,19 +3,20 @@ # Author: Hari Sekhon # Date: 2018-05-22 11:15:16 +0100 (Tue, 22 May 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Mac OS X - Homebrew Package Requirements # ============================================================================ # +# installed by bash-tools submodule now # for anonymize_parallel.sh -parallel +#parallel parquet-tools snappy diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh new file mode 100755 index 000000000..b01a77d04 --- /dev/null +++ b/setup/ci_bootstrap.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-06-02 17:43:35 +0100 (Tue, 02 Jun 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# Designed to bootstrap all CI systems with retries to make sure the networking, package lists and package repos works before proceeding +# +# Minimizes CI build failures due to temporary networking blips, which happens more often than you would think when you have a large number of CI builds across a lot of disparate systems + +set -eu +[ -n "${DEBUG:-}" ] && set -x + +max_tries=10 +interval=60 # secs + +sudo="" +# EUID undefined in posix sh +#[ $EUID = 0 ] || sudo=sudo +[ "$(whoami)" = root ] || sudo=sudo + +retry(){ + # no local in posix sh + count=0 + while true; do + # no let or bare (()) in posix sh, must discard output rather than execute it + _=$((count+=1)) + printf "%s try %d: " "$(date '+%F %T')" "$count" + echo "$*" + "$@" && + break; + echo + if [ "$count" -ge "$max_tries" ]; then + echo "$count tries failed, aborting..." + exit 1 + fi + echo "sleeping for $interval secs before retrying" + sleep "$interval" + echo + done +} + +if [ "$(uname -s)" = Darwin ]; then + echo "Bootstrapping Mac" + # removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make) + #retry "$srcdir/../install/install_homebrew.sh" + if command -v brew 2>&1; then + # fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds + for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do + if [ -d "$git_root" ]; then + # find out if Homebrew is a shallow git checkout and if so fix it + if [ -f "$(git -C "$git_root" rev-parse --git-dir)/shallow" ] || + [ "$(git -C "$git_root" rev-parse --is-shallow-repository)" = true ]; then + git -C "$git_root" fetch --unshallow + fi + fi + done + retry brew update + fi +elif [ "$(uname -s)" = Linux ]; then + echo "Bootstrapping Linux" + if type apk >/dev/null 2>&1; then + retry $sudo apk update + retry $sudo apk add --no-progress bash git make + elif type apt-get >/dev/null 2>&1; then + opts="-q -o DPkg::Lock::Timeout=1200" + retry $sudo apt-get update $opts + retry $sudo apt-get install $opts -y git make + elif type yum >/dev/null 2>&1; then + #retry $sudo yum makecache + retry $sudo yum install -qy git make + else + echo "Package Manager not found on Linux, cannot bootstrap" + exit 1 + fi +else + echo "Only Mac & Linux are supported for conveniently bootstrapping all install scripts at this time" + exit 1 +fi + +#retry make init + +# not calling make because in some CI systems we call 'make ci' which includes retries but in others with more restrictive build minutes we only run 'make' for a single shot build +# +#make diff --git a/setup/ci_git_set_dir_safe.sh b/setup/ci_git_set_dir_safe.sh new file mode 100755 index 000000000..66a53903a --- /dev/null +++ b/setup/ci_git_set_dir_safe.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2022-08-03 20:07:09 +0100 (Wed, 03 Aug 2022) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# Necessary for some CI/CD systems like Azure DevOps Pipelines which have incorrect ownership on the git checkout dir triggering this error: +# +# fatal: detected dubious ownership in repository at '/code/sql' + +# standalone script without lib dependency so it can be called directly from bootstrapped CI before submodules, since that is the exact problem that needs to be solved to allow CI/CD systems with incorrect ownership of the checkout directory to be able to checkout the necessary git submodules + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +dir="${1:-$srcdir/..}" + +cd "$dir" + +echo "Setting directory as safe: $PWD" +git config --global --add safe.directory "$PWD" + +while read -r submodule_dir; do + dir="$PWD/$submodule_dir" + echo "Setting directory as safe: $dir" + git config --global --add safe.directory "$dir" +done < <(git submodule | awk '{print $2}') + +echo "Done" diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt index a2e311a6b..015439134 100644 --- a/setup/deb-packages-dev.txt +++ b/setup/deb-packages-dev.txt @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # @@ -14,6 +14,7 @@ # ============================================================================ # libldap2-dev +libpq-dev # postgres pg_config # needed to build python-snappy for avro module libsnappy-dev diff --git a/setup/deb-packages-optional.txt b/setup/deb-packages-optional.txt index 038ae8802..cd797f747 100644 --- a/setup/deb-packages-optional.txt +++ b/setup/deb-packages-optional.txt @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt index 75196cfb9..da4eea8c8 100644 --- a/setup/deb-packages-pip.txt +++ b/setup/deb-packages-pip.txt @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # @@ -24,6 +24,7 @@ python-ldap python-ldif3 python-numpy python-psutil +python-psycopg2 python-sh python-snappy python-thrift diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt index c605fdd27..4a028e4b1 100644 --- a/setup/deb-packages.txt +++ b/setup/deb-packages.txt @@ -2,33 +2,20 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # # Deb Package Requirements # ============================================================================ # -# needed for tests/test_spark* and ambari_cancel_all_requests.sh -curl - -# either of these should do to ensure ping command is present for find_active_server.py's ping mode -iputils-ping -#inetutils-ping - # for htpasswd for docker registry authenticated checks apache2-utils -# for anonymize_parallel.sh -parallel - -# needed for serf test's 'uptime' command -procps - # installs 343MB of dependencies - install by hand if needed #ffmpeg @@ -41,5 +28,21 @@ procps #which java || $(SUDO) apt-get install -y openjdk-8-jdk || $(SUDO) apt-get install -y openjdk-7-jdk -zip -unzip +# ===================================== +# installed by bash-tools submodule now + +# needed for tests/test_spark* and ambari_cancel_all_requests.sh +#curl + +# either of these should do to ensure ping command is present for find_active_server.py's ping mode +#iputils-ping +##inetutils-ping + +# for anonymize_parallel.sh +#parallel + +# needed for serf test's 'uptime' command +#procps + +#zip +#unzip diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json new file mode 100644 index 000000000..ce63bf63d --- /dev/null +++ b/setup/gocd_config_repo.json @@ -0,0 +1,26 @@ +{ + "id": "DevOps-Python-tools", + "plugin_id": "yaml.config.plugin", + "material": { + "type": "git", + "attributes": { + "url": "https://github.com/HariSekhon/DevOps-Python-tools", + "branch": "master", + "auto_update": true + } + }, + "configuration": [ + { + "key": "file_pattern", + "value": "cicd/*.gocd.y*ml" + } + ], + "rules": [ + { + "directive": "allow", + "action": "*", + "type": "*", + "resource": "*" + } + ] +} diff --git a/setup/jenkins-job.xml b/setup/jenkins-job.xml new file mode 100644 index 000000000..2c9da2b6b --- /dev/null +++ b/setup/jenkins-job.xml @@ -0,0 +1,57 @@ + + + + + + + + hudson.triggers.SCMTrigger + hudson.triggers.TimerTrigger + + + + + + + false + + + + https://github.com/HariSekhon/DevOps-Python-tools/ + + + + + + H 10 * * 1-5 + + + H/2 * * * * + false + + + + + + + 2 + + + https://github.com/HariSekhon/DevOps-Python-tools + + + + + */master + + + false + + + + Jenkinsfile + true + + + false + \ No newline at end of file diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt index a0015d809..f36770476 100644 --- a/setup/rpm-packages-dev.txt +++ b/setup/rpm-packages-dev.txt @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # @@ -17,6 +17,9 @@ gcc-c++ # needed to build python-krbV and cloudera/thrift_sasl cyrus-sasl-devel krb5-devel +openldap-devel openssl-devel + +# moved to optional to account for changed package names on CentOS 8 # needed to build python-snappy for avro module -snappy-devel +#snappy-devel diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt index f94b6019d..b1bb97d9e 100644 --- a/setup/rpm-packages-optional.txt +++ b/setup/rpm-packages-optional.txt @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # @@ -14,3 +14,14 @@ # ============================================================================ # yamllint + +# CentOS <= 7 +snappy-devel +# CentOS 8 +csnappy-devel + +# postgres pg_config +# CentOS 8 +libpq-devel +# CentOS 7 +postgresql-devel diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt index db112cbc7..2a33f4bc5 100644 --- a/setup/rpm-packages-pip.txt +++ b/setup/rpm-packages-pip.txt @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # @@ -25,3 +25,8 @@ python2-psutil #python-flask #python-markupsafe #python2-bitarray + +python2-psycopg2 +python3-psycopg2 + +python3-snappy diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt index 281cfb022..7f6263457 100644 --- a/setup/rpm-packages.txt +++ b/setup/rpm-packages.txt @@ -2,11 +2,11 @@ # Author: Hari Sekhon # Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying LICENSE file # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # @@ -15,21 +15,24 @@ java -# needed for tests/test_spark* and ambari_cancel_all_requests.sh -curl - -# for ping mode of find_active_server.py -iputils - # for htpasswd for docker registry authenticated checks -https-tools +httpd-tools # needed to build pyhs2 # libgsasl-devel saslwrapper-devel #cyrus-sasl-devel +# ===================================== +# installed by bash-tools submodule now + +# needed for tests/test_spark* and ambari_cancel_all_requests.sh +#curl + +# for ping mode of find_active_server.py +#iputils + # for anonymize_parallel.sh -parallel +#parallel -unzip -zip +#unzip +#zip diff --git a/sonar-project.properties b/sonar-project.properties index 75e1bf83c..319cc6897 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,32 +1,42 @@ # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon -# Date: 2016-07-19 18:18:27 +0100 (Tue, 19 Jul 2016) +# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # -sonar.projectName=DevOps Python Tools -sonar.projectKey=pytools +# ============================================================================ # +# S o n a r Q u b e +# ============================================================================ # + +sonar.host.url=https://sonarcloud.io + +# Required metadata +sonar.organization=harisekhon +sonar.projectName=DevOps-Python-tools +sonar.projectKey=HariSekhon_DevOps-Python-tools sonar.projectVersion=1.0 -sonar.projectDescription=Python / Jython Tools +sonar.projectDescription=DevOps-Python-tools -sonar.links.homepage=https://github.com/harisekhon/devops-python-tools -sonar.links.scm=https://github.com/harisekhon/devops-python-tools -sonar.links.issue=https://github.com/harisekhon/devops-python-tools/issues -sonar.links.ci=https://travis-ci.org/HariSekhon/devops-python-tools +sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools +sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools +sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues +sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions +# directories to scan (defaults to sonar-project.properties dir otherwise) sonar.sources=. #sonar.language=py sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/tests/spark*/**/* +#sonar.exclusions=**/tests/** +sonar.exclusions=**/zookeeper-*/**/* diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py index 10fd55e88..8528af1a6 100755 --- a/spark_avro_to_parquet.py +++ b/spark_avro_to_parquet.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py index 955b395e7..d41717b4f 100755 --- a/spark_csv_to_avro.py +++ b/spark_csv_to_avro.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -57,14 +57,15 @@ 'com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error -from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import +from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error __author__ = 'Hari Sekhon' -__version__ = '0.8.0' +__version__ = '0.8.1' class SparkCSVToAvro(CLI): @@ -166,7 +167,6 @@ def create_struct(arg): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') @@ -198,5 +198,6 @@ def create_struct(arg): # the databricks avro driver df.write.format('com.databricks.spark.avro').save(avro_dir) + if __name__ == '__main__': SparkCSVToAvro().main() diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py index f3f6909e6..e70f014f6 100755 --- a/spark_csv_to_parquet.py +++ b/spark_csv_to_parquet.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -49,14 +49,15 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-csv_2.11:1.5.0 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error -from pyspark.sql.types import * # pylint: disable=wrong-import-position,import-error,wildcard-import +from pyspark.sql.types import * # lgtm [py/polluting-import] pylint: disable=wrong-import-position,import-error,wildcard-import from pyspark.sql.types import StructType, StructField # pylint: disable=wrong-import-position,import-error __author__ = 'Hari Sekhon' -__version__ = '0.8.0' +__version__ = '0.8.1' class SparkCSVToParquet(CLI): @@ -157,9 +158,7 @@ def create_struct(arg): if not isVersionLax(spark_version): die("Spark version couldn't be determined. " + support_msg('pytools')) - # pylint: disable=invalid-name - - df = None + # pylint: disable=invalid-name if isMinVersion(spark_version, 1.4): if has_header and not schema: log.info('inferring schema from CSV headers') diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py index 40c52ac96..db9d069ed 100755 --- a/spark_json_to_avro.py +++ b/spark_json_to_avro.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error @@ -109,7 +110,6 @@ def run(self): die("Spark version couldn't be determined. " + support_msg('pytools')) # pylint: disable=invalid-name - df = None if isMinVersion(spark_version, 1.4): df = sqlContext.read.json(json_file) else: diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py index 41ab27e0d..6976ce153 100755 --- a/spark_json_to_parquet.py +++ b/spark_json_to_parquet.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -43,6 +43,7 @@ print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) sys.exit(4) pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py index d6e49f060..030816963 100755 --- a/spark_parquet_to_avro.py +++ b/spark_parquet_to_avro.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -52,6 +52,7 @@ os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \ % os.getenv('PYSPARK_SUBMIT_ARGS', '') pyspark_path() +# pylint: disable=import-outside-toplevel from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error diff --git a/sql b/sql new file mode 160000 index 000000000..30d80bbd3 --- /dev/null +++ b/sql @@ -0,0 +1 @@ +Subproject commit 30d80bbd33f0e6c26c230f26575cb671c41c3e7d diff --git a/strip_ansi_escape_codes.py b/strip_ansi_escape_codes.py index 6fc517dc6..24738194a 100755 --- a/strip_ansi_escape_codes.py +++ b/strip_ansi_escape_codes.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2018-09-09 23:06:06 +0100 (Sun, 09 Sep 2018) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # pylint: disable=line-too-long # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/teamcity/.teamcity.vcs.json b/teamcity/.teamcity.vcs.json new file mode 100644 index 000000000..be61f126b --- /dev/null +++ b/teamcity/.teamcity.vcs.json @@ -0,0 +1,57 @@ +{ + "id": "TeamCity", + "name": "TeamCity", + "vcsName": "jetbrains.git", + "href": "/httpAuth/app/rest/vcs-roots/id:TeamCity", + "project": { + "id": "_Root", + "name": "", + "description": "Contains all other projects", + "href": "/httpAuth/app/rest/projects/id:_Root", + "webUrl": "http://localhost:8111/project.html?projectId=_Root" + }, + "properties": { + "count": 9, + "property": [ + { + "name": "agentCleanFilesPolicy", + "value": "ALL_UNTRACKED" + }, + { + "name": "agentCleanPolicy", + "value": "ON_BRANCH_CHANGE" + }, + { + "name": "authMethod", + "value": "ANONYMOUS" + }, + { + "name": "branch", + "value": "master" + }, + { + "name": "ignoreKnownHosts", + "value": "true" + }, + { + "name": "submoduleCheckout", + "value": "CHECKOUT" + }, + { + "name": "url", + "value": "https://github.com/HariSekhon/TeamCity-CI" + }, + { + "name": "useAlternates", + "value": "true" + }, + { + "name": "usernameStyle", + "value": "USERID" + } + ] + }, + "vcsRootInstances": { + "href": "/httpAuth/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)" + } +} diff --git a/teamcity/.teamcity.vcs.oauth.json b/teamcity/.teamcity.vcs.oauth.json new file mode 100644 index 000000000..f6f41ed7b --- /dev/null +++ b/teamcity/.teamcity.vcs.oauth.json @@ -0,0 +1,61 @@ +{ + "id": "TeamCity", + "name": "TeamCity", + "vcsName": "jetbrains.git", + "href": "/httpAuth/app/rest/vcs-roots/id:TeamCity", + "project": { + "id": "_Root", + "name": "", + "description": "Contains all other projects", + "href": "/httpAuth/app/rest/projects/id:_Root", + "webUrl": "http://localhost:8111/project.html?projectId=_Root" + }, + "properties": { + "count": 9, + "property": [ + { + "name": "agentCleanFilesPolicy", + "value": "ALL_UNTRACKED" + }, + { + "name": "agentCleanPolicy", + "value": "ON_BRANCH_CHANGE" + }, + { + "name": "authMethod", + "value": "PASSWORD" + }, + { + "name": "branch", + "value": "master" + }, + { + "name": "ignoreKnownHosts", + "value": "true" + }, + { + "name": "submoduleCheckout", + "value": "CHECKOUT" + }, + { + "name": "url", + "value": "https://github.com/HariSekhon/TeamCity-CI" + }, + { + "name": "useAlternates", + "value": "true" + }, + { + "name": "username", + "value": "HariSekhon" + }, + { + "name": "usernameStyle", + "value": "USERID" + } + ] + }, + "vcsRootInstances": { + "href": "/httpAuth/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)" + } +} diff --git a/teamcity/.teamcity.vcs.ssh.json b/teamcity/.teamcity.vcs.ssh.json new file mode 100644 index 000000000..1b780675a --- /dev/null +++ b/teamcity/.teamcity.vcs.ssh.json @@ -0,0 +1,65 @@ +{ + "id": "TeamCity", + "name": "TeamCity", + "vcsName": "jetbrains.git", + "href": "/app/rest/vcs-roots/id:TeamCity", + "project": { + "id": "_Root", + "name": "", + "description": "Contains all other projects", + "href": "/app/rest/projects/id:_Root", + "webUrl": "http://localhost:8111/project.html?projectId=_Root" + }, + "properties": { + "count": 11, + "property": [ + { + "name": "agentCleanFilesPolicy", + "value": "ALL_UNTRACKED" + }, + { + "name": "agentCleanPolicy", + "value": "ON_BRANCH_CHANGE" + }, + { + "name": "authMethod", + "value": "TEAMCITY_SSH_KEY" + }, + { + "name": "branch", + "value": "refs/heads/master" + }, + { + "name": "ignoreKnownHosts", + "value": "true" + }, + { + "name": "submoduleCheckout", + "value": "CHECKOUT" + }, + { + "name": "teamcitySshKey", + "value": "VCS SSH Key" + }, + { + "name": "url", + "value": "github.com:HariSekhon/TeamCity-CI" + }, + { + "name": "useAlternates", + "value": "true" + }, + { + "name": "username", + "value": "git" + }, + { + "name": "usernameStyle", + "value": "USERID" + } + ] + }, + "vcsRootInstances": { + "href": "/app/rest/vcs-root-instances?locator=vcsRoot:(id:TeamCity)" + } +} diff --git a/templates b/templates new file mode 160000 index 000000000..f59532dd6 --- /dev/null +++ b/templates @@ -0,0 +1 @@ +Subproject commit f59532dd67a583be3522814a15024a561b12c5ec diff --git a/tests/all.sh b/tests/all.sh index 18dddfc25..dbac7b8fb 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -29,9 +29,11 @@ section "Running PyTools ALL" # runs against . by default cd "$srcdir/.."; -bash-tools/check_all.sh +# shellcheck disable=SC1090 +# has to be included so that isExcluded function is inherited +. "$srcdir/../bash-tools/checks/check_all.sh" -tests/test_yamllint.sh +#tests/test_yamllint.sh # do help afterwards for Spark to be downloaded, and then help will find and use downloaded spark for SPARK_HOME exit 0 @@ -39,4 +41,4 @@ exit 0 # pyspark not found tests/help.sh -bash-tools/run_tests.sh +bash-tools/checks/run_tests.sh diff --git a/tests/check.sh b/tests/check.sh index 07e1bd9c2..b3dba3c12 100755 --- a/tests/check.sh +++ b/tests/check.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-16 16:47:43 +0000 (Sat, 16 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/compile.sh b/tests/compile.sh index e235cfd70..2890b07a3 100755 --- a/tests/compile.sh +++ b/tests/compile.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu @@ -19,17 +19,18 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; -. ./tests/utils.sh +# shellcheck disable=SC1090 +. "$srcdir/utils.sh" hr echo "Compiling all Python files" hr echo -for x in $(find . -iname '*.py' -o -iname '*.jy'); do - isExcluded "$x" && continue - echo "compiling $x" - python -m py_compile $x -done +while read -r filename; do + isExcluded "$filename" && continue + echo "compiling $filename" + python -m py_compile "$filename" +done < <(find . -iname '*.py' -o -iname '*.jy') echo echo diff --git a/tests/data/add_ou.ldif b/tests/data/add_ou.ldif index c175b4bba..c9bc193b2 100644 --- a/tests/data/add_ou.ldif +++ b/tests/data/add_ou.ldif @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2017-08-03 17:44:12 +0200 (Thu, 03 Aug 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # dn: OU=Hadoop-Cluster-1,OU=Hadoop,DC=harisekhon,DC=co,DC=uk diff --git a/tests/data/ldap_download.sh b/tests/data/ldap_download.sh index 1eb3dcd6f..8c7f4a158 100755 --- a/tests/data/ldap_download.sh +++ b/tests/data/ldap_download.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019) # -# https://github.com/harisekhon/pytools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/data/ldap_upload.sh b/tests/data/ldap_upload.sh index 0ff49ca02..60cdeb5ab 100755 --- a/tests/data/ldap_upload.sh +++ b/tests/data/ldap_upload.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019) # -# https://github.com/harisekhon/pytools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/data/uk_marriage_rates_2022.xslx b/tests/data/uk_marriage_rates_2022.xslx new file mode 100644 index 000000000..54efc3321 Binary files /dev/null and b/tests/data/uk_marriage_rates_2022.xslx differ diff --git a/tests/docker/apache-drill-docker-compose.yml b/tests/docker/apache-drill-docker-compose.yml index 50c1de358..d5ede5b8b 100644 --- a/tests/docker/apache-drill-docker-compose.yml +++ b/tests/docker/apache-drill-docker-compose.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2016-12-09 16:18:36 +0000 (Fri, 09 Dec 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.2' diff --git a/tests/docker/common.yml b/tests/docker/common.yml index 4c4236d1c..0b2a2d133 100644 --- a/tests/docker/common.yml +++ b/tests/docker/common.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2016-12-09 15:16:43 +0000 (Fri, 09 Dec 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.2' diff --git a/tests/docker/elasticsearch-docker-compose.yml b/tests/docker/elasticsearch-docker-compose.yml index bd6d9c190..4838ee391 100644 --- a/tests/docker/elasticsearch-docker-compose.yml +++ b/tests/docker/elasticsearch-docker-compose.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.2' diff --git a/tests/docker/elasticsearch-elastic.co-docker-compose.yml b/tests/docker/elasticsearch-elastic.co-docker-compose.yml index 992448ba6..6b7095ab6 100644 --- a/tests/docker/elasticsearch-elastic.co-docker-compose.yml +++ b/tests/docker/elasticsearch-elastic.co-docker-compose.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.2' diff --git a/tests/docker/hadoop-docker-compose.yml b/tests/docker/hadoop-docker-compose.yml index bef1fe74b..157d45c63 100644 --- a/tests/docker/hadoop-docker-compose.yml +++ b/tests/docker/hadoop-docker-compose.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2016-12-09 21:25:07 +0000 (Fri, 09 Dec 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.2' diff --git a/tests/docker/hbase-docker-compose.yml b/tests/docker/hbase-docker-compose.yml index 04029f2bb..8ab70732b 100644 --- a/tests/docker/hbase-docker-compose.yml +++ b/tests/docker/hbase-docker-compose.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2016-12-09 22:13:19 +0000 (Fri, 09 Dec 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.2' diff --git a/tests/docker/presto-dev-docker-compose.yml b/tests/docker/presto-dev-docker-compose.yml index 835b126f1..aaf4872f4 100644 --- a/tests/docker/presto-dev-docker-compose.yml +++ b/tests/docker/presto-dev-docker-compose.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.1' diff --git a/tests/docker/presto-docker-compose.yml b/tests/docker/presto-docker-compose.yml index bb09ee3ed..2b9cc99ed 100644 --- a/tests/docker/presto-docker-compose.yml +++ b/tests/docker/presto-docker-compose.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.1' diff --git a/tests/docker/registry-docker-compose.yml b/tests/docker/registry-docker-compose.yml index 7a66c2e3f..5fae25ab1 100644 --- a/tests/docker/registry-docker-compose.yml +++ b/tests/docker/registry-docker-compose.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2017-09-12 17:27:50 +0200 (Tue, 12 Sep 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.2' diff --git a/tests/docker/solrcloud-docker-compose.yml b/tests/docker/solrcloud-docker-compose.yml index 7799417a7..9cf4885fa 100644 --- a/tests/docker/solrcloud-docker-compose.yml +++ b/tests/docker/solrcloud-docker-compose.yml @@ -3,13 +3,13 @@ # Author: Hari Sekhon # Date: 2016-12-13 14:01:04 +0000 (Tue, 13 Dec 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # version: '2.2' diff --git a/tests/excluded.sh b/tests/excluded.sh index 68f62c58f..7535d3be6 100755 --- a/tests/excluded.sh +++ b/tests/excluded.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # intended only to be sourced by utils.sh @@ -22,8 +22,11 @@ set -eu isExcluded(){ local prog="$1" + # shellcheck disable=SC2049 [[ "$prog" =~ ^\* ]] && return 0 - [[ "$prog" =~ spark_.*.py ]] && return 0 + [[ "$prog" =~ spark_.*\.py ]] && return 0 + [[ "$prog" =~ \.jy ]] && return 0 + [[ "$prog" =~ hdfs_find_replication_factor_1\.py ]] && return 0 # python-krbV doesn't build on Python 3 #[[ $prog =~ ipython-notebook ]] && return 0 # this external git check is expensive, skip it when in CI as using fresh git checkouts is_CI && return 1 diff --git a/tests/help.sh b/tests/help.sh index 7b4edd786..09b48dfe0 100755 --- a/tests/help.sh +++ b/tests/help.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu @@ -33,10 +33,8 @@ for x in ${@:-$(echo ./*.py 2>/dev/null)}; do echo; hr if [ $status = 0 ]; then [[ "$x" =~ ambari_blueprints.py$ ]] && continue - [[ "$x" =~ (hive|impala)_schemas_csv.py$ ]] && continue - [[ "$x" =~ (hive|impala)_foreach_table.py$ ]] && continue - [[ "$x" =~ (hive|impala)_tables_row_counts.py$ ]] && continue [[ "$x" =~ pythonpath.py$ ]] && continue + [[ "$x" =~ aws_s3_presign.py$ ]] && continue elif [ $status = 1 ]; then if [[ "$x" =~ hdfs_find_replication_factor_1.py$ ]] && ! python -c 'import krbV'; then # best effort, not available on Mac any more diff --git a/tests/python3.sh b/tests/python3.sh index 9cd102ac1..b231f8df4 100755 --- a/tests/python3.sh +++ b/tests/python3.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/syntax.sh b/tests/syntax.sh index ea6b33a6a..5299ce323 100755 --- a/tests/syntax.sh +++ b/tests/syntax.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/test_anonymize.py b/tests/test_anonymize.py index 8d67fcf36..4bd3aebc0 100755 --- a/tests/test_anonymize.py +++ b/tests/test_anonymize.py @@ -34,15 +34,20 @@ def run(): global dest # pylint: disable=global-statement src = {int(k) : v for k, v in src.items()} dest = {int(k) : v for k, v in dest.items()} - src_keys = [key for key in sorted(src)] # pylint: disable=redefined-outer-name - test_input = '\n'.join([src[key] for key in src_keys]) + src_keys = sorted(src) + test_input = '\n'.join([src[_] for _ in src_keys]) print('running anonymize tests using: {} {}'.format(anonymize, args)) - cmd = [anonymize] + [_ for _ in args.split()] + cmd = [anonymize] + args.split() process = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE) + # encode as bytes for Python 3 :-/ + test_input = str.encode(test_input, 'utf-8') (stdout, _) = process.communicate(input=test_input) index = 0 - for line in stdout.split('\n'): # pylint: disable=redefined-outer-name + # convert bytes to string + stdout = stdout.decode("utf-8") + # pylint: disable=redefined-outer-name + for line in stdout.split('\n'): key = src_keys[index] _input = src[key] expected = dest[key] diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh index 966142672..9aebe794f 100755 --- a/tests/test_anonymize.sh +++ b/tests/test_anonymize.sh @@ -4,14 +4,14 @@ # Author: Hari Sekhon # Date: 2015-07-28 18:47:41 +0100 (Tue, 28 Jul 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -485,12 +485,17 @@ src[140]='ssh -i myKey -N -L 8888:ec2-1-2-3-4.eu-west-1.compute.amazonaws.com:88 #dest[140]='ssh -i myKey -N -L 8888::8888 @' dest[140]='ssh -i myKey -N -L 8888::8888 @' +# shellcheck disable=SC1117 src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" +# shellcheck disable=SC1117 dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db/
/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory" src[142]="ERROR: AnalysisException: Failed to load metadata for table: 'myCustomerTable2'" dest[142]="ERROR: AnalysisException: Failed to load metadata for table: '
'" +src[143]="PS /pwd> Connect-AppVeyorToComputer -AppVeyorUrl https://ci.appveyor.com -ApiToken a12bcdef3a45b6cdefab" +dest[143]="PS /pwd> Connect-AppVeyorToComputer -AppVeyorUrl https:// -ApiToken " + # TODO: move proxy hosts to host matches and re-enable #src[103]="proxy blah port 8080" @@ -543,6 +548,8 @@ fi # this gives the number of elements and prevents testing the last element(s) if commenting something out in the middle #for (( i = 0 ; i < ${#src[@]} ; i++ )); do run_tests(){ + # expands to the list of indicies in the array, starting at zero - this is easier to work with that ${#src} which is a total + # that is off by one for index usage and doesn't support sparse arrays for any missing/disabled test indicies test_numbers="${*:-${!src[*]}}" for i in $test_numbers; do [ -n "${src[$i]:-}" ] || { echo "code error: src[$i] not defined"; exit 1; } diff --git a/tests/test_apache-drill.sh b/tests/test_apache-drill.sh index c9a675207..af4b663c6 100755 --- a/tests/test_apache-drill.sh +++ b/tests/test_apache-drill.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-26 23:36:03 +0000 (Tue, 26 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -82,6 +82,6 @@ test_apache_drill(){ echo } -startupwait 70 +startupwait 120 run_test_versions "Apache Drill" diff --git a/tests/test_center.sh b/tests/test_center.sh index 0dfd434f8..adb080442 100755 --- a/tests/test_center.sh +++ b/tests/test_center.sh @@ -10,7 +10,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Quick tests, need to replace with testcmd.exp diff --git a/tests/test_docker.sh b/tests/test_docker.sh index 8d00d3be3..37dbb90d5 100755 --- a/tests/test_docker.sh +++ b/tests/test_docker.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-12-08 14:38:37 +0000 (Thu, 08 Dec 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_docker_registry_show_tags.sh b/tests/test_docker_registry_show_tags.sh index bc0612399..e327a3135 100755 --- a/tests/test_docker_registry_show_tags.sh +++ b/tests/test_docker_registry_show_tags.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2017-09-12 17:49:29 +0200 (Tue, 12 Sep 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_dockerfiles_check_git_branches.sh b/tests/test_dockerfiles_check_git_branches.sh index 2529a2c9c..1081c3b98 100755 --- a/tests/test_dockerfiles_check_git_branches.sh +++ b/tests/test_dockerfiles_check_git_branches.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -29,7 +29,7 @@ section "Testing Dockerfiles Check Git branches" if type -P git &>dev/null; then if ! [ -d Dockerfiles ]; then - git clone https://github.com/harisekhon/Dockerfiles + git clone https://github.com/HariSekhon/Dockerfiles else pushd Dockerfiles git pull diff --git a/tests/test_dockerfiles_check_git_tags.sh b/tests/test_dockerfiles_check_git_tags.sh index ada872f3c..a6e0ba12c 100755 --- a/tests/test_dockerfiles_check_git_tags.sh +++ b/tests/test_dockerfiles_check_git_tags.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -29,7 +29,7 @@ section "Testing Dockerfiles Check Git Tags" if type -P git &>dev/null; then if ! [ -d Dockerfiles ]; then - git clone https://github.com/harisekhon/Dockerfiles + git clone https://github.com/HariSekhon/Dockerfiles else pushd Dockerfiles git pull diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh index ee82c5bb8..2a4f9c48e 100755 --- a/tests/test_dockerhub_search.sh +++ b/tests/test_dockerhub_search.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -29,14 +29,14 @@ section "Testing DockerHub Show Tags" check './dockerhub_search.py centos' "DockerHub Search for CentOS" check './dockerhub_search.py harisekhon' "DockerHub Search for harisekhon" -check './dockerhub_search.py harisekhon -n 30' "DockerHub Search for harisekhon -n 30" +check './dockerhub_search.py harisekhon -l 30' "DockerHub Search for harisekhon -l 30" # this no longer works, API must have changed #check './dockerhub_search.py harisekhon/hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub Search for harisekhon/hadoop-dev" # causes IOError: [Errno 32] Broken pipe #unset PYTHONUNBUFFERED # shellcheck disable=SC2016 -check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting" +check '[ "$(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep -c "^harisekhon/[A-Za-z0-9_-]*$")" = 40 ]' "DockerHub Search quiet mode for shell scripting" echo echo diff --git a/tests/test_dockerhub_show_tags.sh b/tests/test_dockerhub_show_tags.sh index 8d052f174..dba64e816 100755 --- a/tests/test_dockerhub_show_tags.sh +++ b/tests/test_dockerhub_show_tags.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_elasticsearch.sh b/tests/test_elasticsearch.sh index 5abd6e53f..17cd5260d 100755 --- a/tests/test_elasticsearch.sh +++ b/tests/test_elasticsearch.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_find_active_server.sh b/tests/test_find_active_server.sh index 1bde9e0e6..615f806d4 100755 --- a/tests/test_find_active_server.sh +++ b/tests/test_find_active_server.sh @@ -10,7 +10,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -144,7 +144,7 @@ ERRCODE=1 run_grep "^NO_AVAILABLE_SERVER$" ./find_active_server.py --https local echo "testing https with url path and regex matching:" echo -run_grep "^github.com$" ./find_active_server.py $opts --https $WEBSITE2 github.com -u /harisekhon --regex 'python-tools' +run_grep "^github.com$" ./find_active_server.py $opts --https $WEBSITE2 github.com -u /harisekhon --regex '(?i)python-tools' # ============================================================================ # diff --git a/tests/test_find_duplicate_files.sh b/tests/test_find_duplicate_files.sh index 9a4e951c9..ade1eeb00 100755 --- a/tests/test_find_duplicate_files.sh +++ b/tests/test_find_duplicate_files.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-08-14 20:42:01 +0100 (Sun, 14 Aug 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -110,10 +110,10 @@ for testdir in "$testdir1" "$testdir2"; do echo "now check the file basename matches on 'est'": run_fail 4 ./find_duplicate_files.py --regex 'est' "$testdir" "$testdir1" --quiet - echo "now check the file basename matches with specified capture subset '(est)\d'": + echo "now check the file basename matches with specified capture subset '(est)\\d'": run_fail 4 ./find_duplicate_files.py --regex '(est)\d' "$testdir" "$testdir1" - echo "now check the file basename doesn't match when the capture includes differing numbers 'est\d'": + echo "now check the file basename doesn't match when the capture includes differing numbers 'est\\d'": run ./find_duplicate_files.py --regex 'est\d' "$testdir" "$testdir1" rm "$testdir/test2.txt" diff --git a/tests/test_find_python_library_path.sh b/tests/test_find_python_library_path.sh index 6a13f5779..7cdc1f43f 100755 --- a/tests/test_find_python_library_path.sh +++ b/tests/test_find_python_library_path.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2019-09-27 # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/test_getent.sh b/tests/test_getent.sh index 26a7aa2b5..6afc48810 100755 --- a/tests/test_getent.sh +++ b/tests/test_getent.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-11-20 15:35:37 +0000 (Sun, 20 Nov 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_git_check_branches_upstream.sh b/tests/test_git_check_branches_upstream.sh index 35a818eb8..8459249bc 100755 --- a/tests/test_git_check_branches_upstream.sh +++ b/tests/test_git_check_branches_upstream.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -29,7 +29,7 @@ section "Testing Git check branches upstream" if type -P git &>/dev/null; then if ! [ -d Dockerfiles ]; then - git clone https://github.com/harisekhon/Dockerfiles + git clone https://github.com/HariSekhon/Dockerfiles else pushd Dockerfiles git pull diff --git a/tests/test_hadoop.sh b/tests/test_hadoop.sh index c2ebedfe9..479789e1f 100755 --- a/tests/test_hadoop.sh +++ b/tests/test_hadoop.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-05-06 12:12:15 +0100 (Fri, 06 May 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_hbase.sh b/tests/test_hbase.sh index 28186c242..4727347a6 100755 --- a/tests/test_hbase.sh +++ b/tests/test_hbase.sh @@ -4,14 +4,14 @@ # Author: Hari Sekhon # Date: 2016-05-06 12:12:15 +0100 (Fri, 06 May 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_headtail.sh b/tests/test_headtail.sh index 8f7ceb05b..2dcfb599c 100755 --- a/tests/test_headtail.sh +++ b/tests/test_headtail.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_json.sh b/tests/test_json.sh index 9e86b0984..34ab409d9 100755 --- a/tests/test_json.sh +++ b/tests/test_json.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/test_json_docs_to_bulk_multiline.sh b/tests/test_json_docs_to_bulk_multiline.sh index 2e81b392a..53d08f35f 100755 --- a/tests/test_json_docs_to_bulk_multiline.sh +++ b/tests/test_json_docs_to_bulk_multiline.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2017-07-30 14:30:00 +0200 (Sun, 30 Jul 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh index 0b20f8609..8b66faa03 100755 --- a/tests/test_json_to_xml.sh +++ b/tests/test_json_to_xml.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_json_to_yaml.sh b/tests/test_json_to_yaml.sh index fc2b58868..4aa156043 100755 --- a/tests/test_json_to_yaml.sh +++ b/tests/test_json_to_yaml.sh @@ -4,31 +4,29 @@ # Author: Hari Sekhon # Date: 2019-12-19 18:04:15 +0000 (Thu, 19 Dec 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x srcdir="$(cd "$(dirname "$0")" && pwd)" -cd "$srcdir"; +cd "$srcdir/.."; -# shellcheck disable=SC1091 -. utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/utils.sh" -# shellcheck disable=SC1091 -. ../bash-tools/lib/utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/../bash-tools/lib/utils.sh" section "JSON => YAML" -cd .. - tmpfile="$(mktemp json_to_yaml_test.XXXXX.yml)" #echo "tmpfile is $tmpfile" diff --git a/tests/test_opentsdb.sh b/tests/test_opentsdb.sh index ac98e609a..bbc59a11f 100755 --- a/tests/test_opentsdb.sh +++ b/tests/test_opentsdb.sh @@ -4,14 +4,14 @@ # Author: Hari Sekhon # Date: 2016-10-10 11:54:19 +0100 (Mon, 10 Oct 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_presto.sh b/tests/test_presto.sh index ba9add032..c031d5697 100755 --- a/tests/test_presto.sh +++ b/tests/test_presto.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2017-09-22 17:01:38 +0200 (Fri, 22 Sep 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_quay_show_tags.sh b/tests/test_quay_show_tags.sh index 6765bc9e9..96f451c23 100755 --- a/tests/test_quay_show_tags.sh +++ b/tests/test_quay_show_tags.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_serf_event_handler.sh b/tests/test_serf_event_handler.sh index eee228571..d16c8e2f7 100755 --- a/tests/test_serf_event_handler.sh +++ b/tests/test_serf_event_handler.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_solrcloud.sh b/tests/test_solrcloud.sh index 31df34f43..2f0f69196 100755 --- a/tests/test_solrcloud.sh +++ b/tests/test_solrcloud.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-01-22 21:13:49 +0000 (Fri, 22 Jan 2016) # -# https://github.com/harisekhon/nagios-plugins +# https://github.com/HariSekhon/Nagios-Plugins # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_spark_csv_to_avro.sh b/tests/test_spark_csv_to_avro.sh index dc43c31d8..5765ba160 100755 --- a/tests/test_spark_csv_to_avro.sh +++ b/tests/test_spark_csv_to_avro.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/test_spark_csv_to_parquet.sh b/tests/test_spark_csv_to_parquet.sh index 7f95341e1..aa2b8d2dd 100755 --- a/tests/test_spark_csv_to_parquet.sh +++ b/tests/test_spark_csv_to_parquet.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/test_spark_json_to_avro.sh b/tests/test_spark_json_to_avro.sh index d6506a50c..bb281026f 100755 --- a/tests/test_spark_json_to_avro.sh +++ b/tests/test_spark_json_to_avro.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/test_spark_json_to_parquet.sh b/tests/test_spark_json_to_parquet.sh index 9a4a8bc37..dc3e2fef4 100755 --- a/tests/test_spark_json_to_parquet.sh +++ b/tests/test_spark_json_to_parquet.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/test_spark_z_avro_to_parquet.sh b/tests/test_spark_z_avro_to_parquet.sh index 02fc1ea26..b6c3ff7cb 100755 --- a/tests/test_spark_z_avro_to_parquet.sh +++ b/tests/test_spark_z_avro_to_parquet.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/test_spark_z_parquet_to_avro.sh b/tests/test_spark_z_parquet_to_avro.sh index 6e88f0cfc..9e02af7ed 100755 --- a/tests/test_spark_z_parquet_to_avro.sh +++ b/tests/test_spark_z_parquet_to_avro.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/tests/test_timeout.sh b/tests/test_timeout.sh index 26d38d589..5d46e4bd6 100755 --- a/tests/test_timeout.sh +++ b/tests/test_timeout.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-02-14 16:16:00 +0000 (Sun, 14 Feb 2016) # -# https://github.com/harisekhon/bash-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_avro.sh b/tests/test_validate_avro.sh index d8dab1d88..b8bf5fbad 100755 --- a/tests/test_validate_avro.sh +++ b/tests/test_validate_avro.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_cson.sh b/tests/test_validate_cson.sh index 20b9d332b..966a6d201 100755 --- a/tests/test_validate_cson.sh +++ b/tests/test_validate_cson.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_csv.sh b/tests/test_validate_csv.sh index 680b63390..1747bebcc 100755 --- a/tests/test_validate_csv.sh +++ b/tests/test_validate_csv.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh index f2282be6a..72349b24d 100755 --- a/tests/test_validate_ini.sh +++ b/tests/test_validate_ini.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_json.sh b/tests/test_validate_json.sh index 39b24306b..b92d16c2c 100755 --- a/tests/test_validate_json.sh +++ b/tests/test_validate_json.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_ldap_ldif.sh b/tests/test_validate_ldap_ldif.sh index 53af23074..267269b32 100755 --- a/tests/test_validate_ldap_ldif.sh +++ b/tests/test_validate_ldap_ldif.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2017-08-20 12:56:43 +0100 (Sun, 20 Aug 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh index 7302ca55d..3493bae6d 100755 --- a/tests/test_validate_multimedia.sh +++ b/tests/test_validate_multimedia.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2016-05-01 20:46:42 +0100 (Sun, 01 May 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail @@ -32,7 +32,7 @@ if ! type -P ffmpeg &>/dev/null; then echo "WARNING: ffmpeg not installed, skipping validate_multimedia.py tests" exit 0 if type -P apt-get &>/dev/null; then - sudo apt-get install -y ffmpeg + sudo apt-get install -o DPkg::Lock::Timeout=1200 -y ffmpeg elif type -P yum &>/dev/null; then echo "WARNING: cannot auto-install ffmpeg on RHEL/CentOS, the 3rd party repos and deps are seriously broken" fi diff --git a/tests/test_validate_parquet.sh b/tests/test_validate_parquet.sh index d75942e84..44b0c8f7d 100755 --- a/tests/test_validate_parquet.sh +++ b/tests/test_validate_parquet.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_toml.sh b/tests/test_validate_toml.sh index 3dd37ed60..c20a38cf6 100755 --- a/tests/test_validate_toml.sh +++ b/tests/test_validate_toml.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_xml.sh b/tests/test_validate_xml.sh index 37b1ee106..7fe631f3b 100755 --- a/tests/test_validate_xml.sh +++ b/tests/test_validate_xml.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_validate_yaml.sh b/tests/test_validate_yaml.sh index 098cf827d..abd4343ca 100755 --- a/tests/test_validate_yaml.sh +++ b/tests/test_validate_yaml.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/test_welcome.sh b/tests/test_welcome.sh index 1be838108..0a13136a7 100755 --- a/tests/test_welcome.sh +++ b/tests/test_welcome.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu @@ -20,6 +20,14 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$srcdir/.."; # shellcheck disable=SC1091 -#. ./tests/utils.sh +. ./tests/utils.sh + +# Fedora doesn't have /var/log/wtmp +if ! [ -f /var/log/wtmp ]; then + echo "/var/log/wtmp doesn't exist, touching..." + # assigned in utils.sh + # shellcheck disable=SC2154 + $sudo touch /var/log/wtmp || : +fi ./welcome.py "$@" diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh index 8d01e3750..6bd9cf731 100755 --- a/tests/test_xml_to_json.sh +++ b/tests/test_xml_to_json.sh @@ -4,30 +4,30 @@ # Author: Hari Sekhon # Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(cd "$(dirname "$0")" && pwd)" +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$srcdir"; +cd "$srcdir" -# shellcheck disable=SC1091 -. utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/utils.sh" -# shellcheck disable=SC1091 -. ../bash-tools/lib/utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/../bash-tools/lib/utils.sh" section "XML => JSON" -cd .. +cd "$srcdir/.." testdata="tests/data/simple.xml" diff --git a/tests/test_xml_to_yaml.sh b/tests/test_xml_to_yaml.sh index 6f93a6ddf..4b3d72da0 100755 --- a/tests/test_xml_to_yaml.sh +++ b/tests/test_xml_to_yaml.sh @@ -4,30 +4,30 @@ # Author: Hari Sekhon # Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail [ -n "${DEBUG:-}" ] && set -x -srcdir="$(cd "$(dirname "$0")" && pwd)" +srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$srcdir"; +cd "$srcdir" -# shellcheck disable=SC1091 -. utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/utils.sh" -# shellcheck disable=SC1091 -. ../bash-tools/lib/utils.sh +# shellcheck disable=SC1091,SC1090 +. "$srcdir/../bash-tools/lib/utils.sh" section "XML => YAML" -cd .. +cd "$srcdir/.." testdata="tests/data/simple.xml" diff --git a/tests/test_yamllint.sh b/tests/test_yamllint.sh index 31d808ff6..90c983ce4 100755 --- a/tests/test_yamllint.sh +++ b/tests/test_yamllint.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2019-02-26 14:40:53 +0000 (Tue, 26 Feb 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail diff --git a/tests/utils.sh b/tests/utils.sh index 57c32ca46..eddb33d01 100755 --- a/tests/utils.sh +++ b/tests/utils.sh @@ -5,13 +5,13 @@ # Author: Hari Sekhon # Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -eu diff --git a/timeout.py b/timeout.py index b274872a3..831d2e99f 100755 --- a/timeout.py +++ b/timeout.py @@ -1,16 +1,16 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-02-14 15:46:37 +0000 (Sun, 14 Feb 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # pylint: disable=line-too-long # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/travis_debug_session.py b/travis_debug_session.py index 6812eeead..aafebf35e 100755 --- a/travis_debug_session.py +++ b/travis_debug_session.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-08-10 18:18:03 +0100 (Wed, 10 Aug 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -48,6 +48,7 @@ import sys import time import traceback +import git try: import requests except ImportError: @@ -68,7 +69,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.8.4' +__version__ = '0.9.1' class TravisDebugSession(CLI): @@ -151,10 +152,24 @@ def process_options(self): self.repo = travis_user + self.repo validate_chars(self.repo, 'repo', r'\/\w\.-') else: - self.usage('--job-id / --repo not specified') - validate_alnum(self.travis_token, 'travis token') + self.repo = self.get_local_repo_name() + if not self.repo: + self.usage('--job-id / --repo not specified') + validate_alnum(self.travis_token, 'travis token', is_secret=True) self.headers['Authorization'] = 'token {0}'.format(self.travis_token) + @staticmethod + def get_local_repo_name(): + try: + _ = git.Repo('.') + for remote in _.remotes: + for url in remote.urls: + repo = '/'.join(url.split('/')[-2:]) + log.debug('determined repo to be {} from remotes'.format(repo)) + return repo + except git.InvalidGitRepositoryError: + log.debug('failed to determine git repository locally: %s', _) + def run(self): if not self.job_id: if self.repo: diff --git a/travis_last_log.py b/travis_last_log.py index b9b6843f2..82b7493e5 100755 --- a/travis_last_log.py +++ b/travis_last_log.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-08-10 18:18:03 +0100 (Wed, 10 Aug 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -61,6 +61,7 @@ #import string import sys import traceback +import git srcdir = os.path.abspath(os.path.dirname(__file__)) libdir = os.path.join(srcdir, 'pylib') sys.path.append(libdir) @@ -76,7 +77,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.5.0' +__version__ = '0.6.1' class TravisLastBuildLog(CLI): @@ -147,8 +148,10 @@ def process_options(self): self.repo = travis_user + self.repo validate_chars(self.repo, 'repo', r'\/\w\.-') else: - self.usage('--job-id / --repo not specified') - validate_alnum(self.travis_token, 'travis token') + self.repo = self.get_local_repo_name() + if not self.repo: + self.usage('--job-id / --repo not specified') + validate_alnum(self.travis_token, 'travis token', is_secret=True) self.headers['Authorization'] = 'token {0}'.format(self.travis_token) self.num = self.get_opt('num') validate_int(self.num, 'num', 1) @@ -163,6 +166,18 @@ def process_options(self): #if not self.color and not (sys.__stdin__.isatty() and sys.__stdout__.isatty()): # self.plaintext = True + @staticmethod + def get_local_repo_name(): + try: + _ = git.Repo('.') + for remote in _.remotes: + for url in remote.urls: + repo = '/'.join(url.split('/')[-2:]) + log.debug('determined repo to be {} from remotes'.format(repo)) + return repo + except git.InvalidGitRepositoryError: + log.debug('failed to determine git repository locally: %s', _) + def run(self): if self.job_id: self.print_log(job_id=self.job_id) diff --git a/urldecode.py b/urldecode.py new file mode 100755 index 000000000..87319d3be --- /dev/null +++ b/urldecode.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Tool to url decode text from standard input or a text argument + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon import CLI + from harisekhon.utils import isPythonMinVersion +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +# pylint: disable=no-name-in-module,import-error +if isPythonMinVersion(3): + from urllib.parse import unquote_plus as unquote +else: + from urllib import unquote_plus as unquote + +__author__ = 'Hari Sekhon' +__version__ = '0.1.1' + + +class URLDecode(CLI): + + def __init__(self): + # Python 2.x + super(URLDecode, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = None + + def run(self): + if len(sys.argv) > 1: + for arg in sys.argv[1:]: + self.decode(arg) + else: + # buffered - Control-D char meshes with late output + #for line in sys.stdin: + while True: + line = sys.stdin.readline() + if not line: + break + line = line.rstrip('\n').rstrip('\r') + self.decode(line) + + @staticmethod + def decode(string): + print(unquote(string)) + sys.stdout.flush() + + +if __name__ == '__main__': + URLDecode().main() diff --git a/urlencode.py b/urlencode.py new file mode 100755 index 000000000..6ca907f15 --- /dev/null +++ b/urlencode.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Tool to url encode text from standard input or a text argument + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import os +import sys +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon import CLI + from harisekhon.utils import isPythonMinVersion +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +# pylint: disable=no-name-in-module,import-error +if isPythonMinVersion(3): + from urllib.parse import quote_plus as quote +else: + from urllib import quote_plus as quote + +__author__ = 'Hari Sekhon' +__version__ = '0.1.1' + + +class URLEncode(CLI): + + def __init__(self): + # Python 2.x + super(URLEncode, self).__init__() + # Python 3.x + # super().__init__() + self.timeout_default = None + + def run(self): + if len(sys.argv) > 1: + for arg in sys.argv[1:]: + self.encode(arg) + else: + # buffered - Control-D char meshes with late output + #for line in sys.stdin: + while True: + line = sys.stdin.readline() + if not line: + break + line = line.rstrip('\n').rstrip('\r') + self.encode(line) + + @staticmethod + def encode(string): + #print(urllib.parse.quote(string)) + print(quote(string)) + sys.stdout.flush() + + +if __name__ == '__main__': + URLEncode().main() diff --git a/validate_avro.py b/validate_avro.py index f8a896466..37eaab848 100755 --- a/validate_avro.py +++ b/validate_avro.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -39,7 +39,7 @@ try: from avro3.datafile import DataFileReader, DataFileException from avro3.io import DatumReader -except: +except Exception: # pylint: disable=broad-except from avro.datafile import DataFileReader, DataFileException from avro.io import DatumReader libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) @@ -55,7 +55,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.9.1' +__version__ = '0.9.2' class AvroValidatorTool(CLI): diff --git a/validate_cson.py b/validate_cson.py index 4287f6d93..6e017ab97 100755 --- a/validate_cson.py +++ b/validate_cson.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-10-07 10:23:31 +0100 (Mon, 07 Oct 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/validate_csv.py b/validate_csv.py index c53d16eb1..bac1d583e 100755 --- a/validate_csv.py +++ b/validate_csv.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/validate_ini.py b/validate_ini.py index 84411046d..b38b3ab6a 100755 --- a/validate_ini.py +++ b/validate_ini.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2017-09-15 15:29:39 +0200 (Fri, 15 Sep 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.12.2' +__version__ = '0.12.3' class IniValidatorTool(CLI): @@ -72,7 +72,7 @@ def __init__(self): self.re_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I) # In Windows ini key cannot contain equals sign = or semicolon ; # key=val or [section] - self.re_ini_section = re.compile(r'^\s*\[([\w=\:\.-]+)\]\s*$') + self.re_ini_section = re.compile(r'^\s*\[([\w\s=\:\.-]+)\]\s*$') self.re_ini_key = re.compile(r'^\s*(?:[^\[;=]+)s*$') # INI value can be anything .* so not regex'ing it self.valid_ini_msg = ' => INI OK' diff --git a/validate_ini2.py b/validate_ini2.py index aaf17f87f..5c41436a4 100755 --- a/validate_ini2.py +++ b/validate_ini2.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2017-09-15 15:29:39 +0200 (Fri, 15 Sep 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/validate_json.py b/validate_json.py index 03cc0aa7a..dbca2c431 100755 --- a/validate_json.py +++ b/validate_json.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -58,7 +58,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '0.11.1' +__version__ = '0.11.2' class JsonValidatorTool(CLI): @@ -210,7 +210,7 @@ def check_json(self, content): return True self.failed = True if not self.passthru: - die(self.self.invalid_json_msg_single_quotes) + die(self.invalid_json_msg_single_quotes) else: log.debug('not valid json') if self.rewind_check_multirecord_json(): diff --git a/validate_ldap_ldif.py b/validate_ldap_ldif.py index 02e3e4bf7..963b725c8 100755 --- a/validate_ldap_ldif.py +++ b/validate_ldap_ldif.py @@ -1,18 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2017-08-04 00:53:10 +0200 (Fri, 04 Aug 2017) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn # and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/validate_multimedia.py b/validate_multimedia.py index 9452fd6c7..0362bb30d 100755 --- a/validate_multimedia.py +++ b/validate_multimedia.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-05-01 20:46:56 +0100 (Sun, 01 May 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -45,7 +45,7 @@ import re import sys import subprocess -from subprocess import CalledProcessError +CalledProcessError = subprocess.CalledProcessError libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) sys.path.append(libdir) try: @@ -164,8 +164,8 @@ def check_path(self, path): die("failed to determine if path '%s' is file or directory" % path) def check_media_file(self, filename): - if self.is_excluded(filename): - return + #if self.is_excluded(filename): + # return valid_media_msg = '%s => OK' % filename invalid_media_msg = '%s => INVALID' % filename cmd = self.validate_cmd diff --git a/validate_parquet.py b/validate_parquet.py index 71894645f..bf6690d06 100755 --- a/validate_parquet.py +++ b/validate_parquet.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/validate_toml.py b/validate_toml.py index b75c2dce0..25628d222 100755 --- a/validate_toml.py +++ b/validate_toml.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-10-07 10:23:31 +0100 (Mon, 07 Oct 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/validate_xml.py b/validate_xml.py index 5620a81e5..c954bac42 100755 --- a/validate_xml.py +++ b/validate_xml.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/validate_yaml.py b/validate_yaml.py index 819943294..75ee4034c 100755 --- a/validate_yaml.py +++ b/validate_yaml.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/welcome.py b/welcome.py index 0d93b836f..d9c1b2546 100755 --- a/welcome.py +++ b/welcome.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2009-12-09 19:58:14 +0000 (Wed, 09 Dec 2009) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ @@ -45,7 +45,7 @@ sys.exit(4) __author__ = 'Hari Sekhon' -__version__ = '2.0.2' +__version__ = '2.0.4' class Welcome(CLI): @@ -59,7 +59,7 @@ def __init__(self): self.timeout_default = 20 @staticmethod - def case_user(user): + def titlecase_user(user): if user == 'root': user = user.upper() elif len(user) < 4 or re.search(r'\d', user): @@ -76,7 +76,7 @@ def construct_msg(self): # print("invalid user '%s' determined from environment variable $USER, failed regex validation" % user) print("invalid user '%s' returned by getpass.getuser(), failed regex validation" % user) sys.exit(ERRORS['CRITICAL']) - user = self.case_user(user) + user = self.titlecase_user(user) msg = 'Welcome %s - ' % user last = '' if which("last"): @@ -121,7 +121,7 @@ def print_welcome(self): print(msg) return try: - charmap = list(string.uppercase + string.lowercase + '@#$%^&*()') + charmap = list(string.ascii_uppercase + string.ascii_lowercase + '@#$%^&*()') # print '', # print('', end='') for i in range(0, len(msg)): diff --git a/xml_to_json.py b/xml_to_json.py index a82ea8adf..14d4cde62 100755 --- a/xml_to_json.py +++ b/xml_to_json.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2016-01-15 00:07:09 +0000 (Fri, 15 Jan 2016) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/xml_to_yaml.py b/xml_to_yaml.py index 6643a970a..47a098243 100755 --- a/xml_to_yaml.py +++ b/xml_to_yaml.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # vim:ts=4:sts=4:sw=4:et # # Author: Hari Sekhon # Date: 2019-12-19 18:19:34 +0000 (Thu, 19 Dec 2019) # -# https://github.com/harisekhon/devops-python-tools +# https://github.com/HariSekhon/DevOps-Python-tools # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # """ diff --git a/yaml_to_json.py b/yaml_to_json.py new file mode 100755 index 000000000..90a5db9b7 --- /dev/null +++ b/yaml_to_json.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# vim:ts=4:sts=4:sw=4:et +# args: ../bash-tools/.gitlab-ci.yml +# +# Author: Hari Sekhon +# Date: 2020-08-18 01:17:56 +0100 (Tue, 18 Aug 2020) +# +# https://github.com/HariSekhon/DevOps-Python-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# to help improve or steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +""" + +Tool to convert YAML to JSON + +Reads any given files as YAML and prints the equivalent JSON to stdout for piping or redirecting to a file. + +Directories if given are detected and recursed, processing all files in the directory tree ending in a +.yml / .yaml suffix. + +Works like a standard unix filter program - if no files are passed as arguments or '-' is passed then reads from +standard input. + +Written to convert .gitlab-ci.yml files to JSON for inputting to the GitLab API for validation +(see gitlab_validate_ci_yml.sh in the DevOps-Bash-tools repo) + +See also: + + yaml2json.sh - https://github.com/HariSekhon/DevOps-Bash-tools + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +#from __future__ import unicode_literals + +import json +import os +import re +import sys +import yaml +libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'pylib')) +sys.path.append(libdir) +try: + # pylint: disable=wrong-import-position + from harisekhon.utils import die, ERRORS, log, log_option + from harisekhon import CLI +except ImportError as _: + print('module import failed: %s' % _, file=sys.stderr) + print("Did you remember to build the project by running 'make'?", file=sys.stderr) + print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr) + sys.exit(4) + +__author__ = 'Hari Sekhon' +__version__ = '0.2.0' + + +class YamlToJson(CLI): + + def __init__(self): + # Python 2.x + super(YamlToJson, self).__init__() + # Python 3.x + # super().__init__() + self.re_yaml_suffix = re.compile(r'.*\.ya?ml$', re.I) + + @staticmethod + def yaml_to_json(content, filepath=None): + try: + _ = yaml.load(content, Loader=yaml.FullLoader) + except (KeyError, ValueError) as _: + file_detail = '' + if filepath is not None: + file_detail = ' in file \'{0}\''.format(filepath) + die("Failed to parse YAML{0}: {1}".format(file_detail, _)) + return json.dumps(_, sys.stdout, indent=4) + + def run(self): + if not self.args: + self.args.append('-') + for arg in self.args: + if arg == '-': + continue + if not os.path.exists(arg): + print("'%s' not found" % arg) + sys.exit(ERRORS['WARNING']) + if os.path.isfile(arg): + log_option('file', arg) + elif os.path.isdir(arg): + log_option('directory', arg) + else: + die("path '%s' could not be determined as either a file or directory" % arg) + for arg in self.args: + self.process_path(arg) + + def process_path(self, path): + if path == '-' or os.path.isfile(path): + self.process_file(path) + elif os.path.isdir(path): + for root, _, files in os.walk(path): + for filename in files: + filepath = os.path.join(root, filename) + if self.re_yaml_suffix.match(filepath): + self.process_file(filepath) + else: + die("failed to determine if path '%s' is a file or directory" % path) + + def process_file(self, filepath): + log.debug('processing filepath \'%s\'', filepath) + if filepath == '-': + filepath = '' + if filepath == '': + print(self.yaml_to_json(sys.stdin.read())) + else: + with open(filepath) as _: + content = _.read() + print(self.yaml_to_json(content, filepath=filepath)) + + +if __name__ == '__main__': + YamlToJson().main()