diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index c490c04ef..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Python CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-python/ for more details -# -version: 2 - -jobs: - build: - branches: - ignore: - - gh-pages - docker: - - image: googleapis/nox:0.17.0 - - image: mysql:5.7 - environment: - MYSQL_ROOT_HOST: "%" - MYSQL_ROOT_PASSWORD: 666666 - ports: - - 3306:3306 - - image: circleci/postgres:9.6 - environment: - POSTGRES_PASSWORD: 666666 - ports: - - 5432:5432 - - working_directory: ~/repo - - steps: - - checkout - - run: - name: Decrypt credentials - command: | - if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then - openssl aes-256-cbc -d -a -k "$GOOGLE_CREDENTIALS_PASSPHRASE" \ - -in tests/system/credentials.json.enc \ - -out $GOOGLE_APPLICATION_CREDENTIALS - else - echo "No credentials. System tests will not run." - fi - - run: - name: Run tests - opencensus - command: | - pip install --upgrade nox - nox -f noxfile.py - - deploy: - name: Push to PyPI (if this is a release tag). - command: scripts/twine_upload.sh - -deployment: - tag_build_for_cci2: - branch: /v([0-9]+)\.([0-9]+)\.([0-9]+)/ - tag: /v([0-9]+)\.([0-9]+)\.([0-9]+)/ - commands: - - true diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 826cd01b7..2e1dc6b60 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,4 @@ # This file controls who is tagged for review for any given pull request. # For anything not explicitly taken by someone else: -* @census-instrumentation/global-owners @c24t @reyang @songy23 +* @census-instrumentation/global-owners @aabmass @hectorhdzg @jeremydvoss @lzchen diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..aa8a05f51 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,61 @@ +name: Build + +on: + push: + branches-ignore: + - 'release/*' + pull_request: + branches: + - master + +jobs: + build: + runs-on: ubuntu-20.04 + env: + # We use these variables to convert between tox and GHA version literals + py35: 3.5 + py36: 3.6 + py37: 3.7 + py38: 3.8 + py39: 3.9 + strategy: + # ensures the entire test matrix is run, even if one permutation fails + fail-fast: false + matrix: + python-version: [py35, py36, py37, py38, py39] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Python ${{ env[matrix.python-version] }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env[matrix.python-version] }} + - name: Install tox + # Pin tox 3 because of https://github.com/rpkilby/tox-factor/issues/18 + run: pip install -U tox==3.27.1 tox-factor + - name: Cache tox environment + uses: actions/cache@v2 + with: + path: .tox + # bump version prefix to fully reset caches + key: v1-tox-${{ matrix.python-version }}-${{ hashFiles('tox.ini', '**/setup.py') }} + - name: run tox + run: tox -f ${{ matrix.python-version }} + build-27: + runs-on: ubuntu-20.04 + container: + image: python:2.7.18-buster + env: + py27: 2.7 + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Install tox + run: pip install -U tox==3.27.1 tox-factor + - name: Cache tox environment + uses: actions/cache@v2 + with: + path: .tox + key: v1-tox-27-${{ hashFiles('tox.ini', '**/setup.py') }} + - name: Run tox for Python 2.7 + run: tox -f py27 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..79f978fb9 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,27 @@ +name: CodeQL Analysis + +on: + workflow_dispatch: + push: + branches-ignore: + - 'release/*' + +jobs: + CodeQL-Build: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: python + + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..0b7fb3d05 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,17 @@ +[settings] +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=79 + +; 3 stands for Vertical Hanging Indent, e.g. +; from third_party import ( +; lib1, +; lib2, +; lib3, +; ) +; docs: https://github.com/timothycrosley/isort#multi-line-output-modes +multi_line_output=3 +known_future_library = six,six.moves,__future__ +known_third_party=azure-core,azure-identity,google,mock,pymysql,sqlalchemy,psycopg2,mysql,requests,django,pytest,grpc,flask,bitarray,prometheus_client,psutil,pymongo,wrapt,thrift,retrying,pyramid,werkzeug,gevent +known_first_party=opencensus \ No newline at end of file diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..e34531789 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,563 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Add files or directories to the excludelist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the excludelist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + # old-raise-syntax, + # backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape, + useless-object-inheritance, + missing-docstring, + too-few-public-methods, + too-many-arguments, + import-error, + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +# enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, while `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package.. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format=LF + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=_, + log, + logger + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +variable-rgx=(([a-z_][a-z0-9_]{1,})|(_[a-z0-9_]*)|(__[a-z][a-z0-9_]+__))$ + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=yes + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library=six + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement. +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception". +overgeneral-exceptions=Exception diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f43b655b0..000000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -dist: xenial -language: python - -python: - - '2.7' - - '3.4' - - '3.5' - - '3.6' - - '3.7' - -install: - - pip install tox-travis - -script: - - tox - - touch docs/.nojekyll - -branches: - only: - - master diff --git a/CHANGELOG.md b/CHANGELOG.md index d2599f325..119d0f3e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,154 @@ ## Unreleased +# 0.11.4 +Released 2024-01-03 +- Changed bit-mapping for `httpx` and `fastapi` integrations +([#1239](https://github.com/census-instrumentation/opencensus-python/pull/1239)) + +# 0.11.3 +Released 2023-09-18 + +- Updated `azure` modules + +# 0.11.2 +Released 2023-03-10 + +- Updated `azure`, `fastapi`,`flask` modules + +# 0.11.1 +Released 2023-01-18 + +- Updated `azure`, `httpx` modules + +# 0.11.0 +Released 2022-08-03 + +- Updated `azure`, `context`, `flask`, `requests` modules + +# 0.10.0 +Released 2022-07-05 + +- Add kwargs to derived gauge +([#1135](https://github.com/census-instrumentation/opencensus-python/pull/1135)) + +# 0.9.0 +Released 2022-04-20 + +- Make sure handler.flush() doesn't deadlock +([#1112](https://github.com/census-instrumentation/opencensus-python/pull/1112)) + +# 0.8.0 +Released 2021-10-05 + +- Added integration tracking functionality, includes `django`, `flask`, `http-lib`, `logging`, `mysql`, `postgresql`, `pymongo`, `pymysql`, `pyramid`, `requests`, `sqlalchemy` modules +([#1065](https://github.com/census-instrumentation/opencensus-python/pull/1065)) +- Support Python 3.8, 3.9 +([#1048](https://github.com/census-instrumentation/opencensus-python/pull/1048)) + +# 0.7.13 +Released 2021-05-13 + +- Updated `azure`, `django`, `flask`, `requests` modules + +# 0.7.12 +Released 2021-01-14 + +- Updated `azure`, `django`, `flask`, `grpc`, `httplib`, `pyramid`, `requests` modules + +# 0.7.11 +Released 2020-10-13 + +- Updated `azure`, `stackdriver` modules + +## 0.7.10 +Released 2020-06-29 + +- Updated `azure` module +([#903](https://github.com/census-instrumentation/opencensus-python/pull/903), + [#925](https://github.com/census-instrumentation/opencensus-python/pull/925)) + +- Updated `stackdriver` module +([#919](https://github.com/census-instrumentation/opencensus-python/pull/919)) + +## 0.7.9 +Released 2020-06-17 + +- Hotfix for breaking change + ([#915](https://github.com/census-instrumentation/opencensus-python/pull/915)) + +## 0.7.8 +Released 2020-06-17 + +- Updated `azure` module + ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903), + [#902](https://github.com/census-instrumentation/opencensus-python/pull/902)) + +## 0.7.7 +Released 2020-02-03 + +- Updated `azure` module +([#837](https://github.com/census-instrumentation/opencensus-python/pull/837), + [#845](https://github.com/census-instrumentation/opencensus-python/pull/845), + [#848](https://github.com/census-instrumentation/opencensus-python/pull/848), + [#851](https://github.com/census-instrumentation/opencensus-python/pull/851)) + +## 0.7.6 +Released 2019-11-26 + +- Initial release for `datadog` module + ([#793](https://github.com/census-instrumentation/opencensus-python/pull/793)) +- Updated `azure` module + ([#789](https://github.com/census-instrumentation/opencensus-python/pull/789), + [#822](https://github.com/census-instrumentation/opencensus-python/pull/822)) + +## 0.7.5 +Released 2019-10-01 + +- Updated `flask` module + ([#781](https://github.com/census-instrumentation/opencensus-python/pull/781)) + +## 0.7.4 +Released 2019-09-30 + +- Updated `azure` module + ([#773](https://github.com/census-instrumentation/opencensus-python/pull/773), + [#767](https://github.com/census-instrumentation/opencensus-python/pull/767)) + +- Updated `django` module + ([#775](https://github.com/census-instrumentation/opencensus-python/pull/775)) + +## 0.7.3 +Released 2019-08-26 + +- Added `http code` to `grpc code` status code mapping on `utils` + ([#746](https://github.com/census-instrumentation/opencensus-python/pull/746)) +- Updated `django`, `flask`, `httplib`, `requests` and `pyramid` modules + ([#755](https://github.com/census-instrumentation/opencensus-python/pull/755)) +- Updated `requests` module + ([#771](https://github.com/census-instrumentation/opencensus-python/pull/771)) + +## 0.7.2 +Released 2019-08-16 + +- Fix GCP resource loading for certain environments + ([#761](https://github.com/census-instrumentation/opencensus-python/pull/761)) + +## 0.7.1 +Released 2019-08-05 + +- Added `set_status` to `span` + ([#738](https://github.com/census-instrumentation/opencensus-python/pull/738)) +- Update released stackdriver exporter version + +## 0.7.0 +Released 2019-07-31 + +- Fix exporting int-valued stats with sum and lastvalue aggregations + ([#696](https://github.com/census-instrumentation/opencensus-python/pull/696)) +- Fix cloud format propagator to use decimal span_id encoding instead of hex + ([#719](https://github.com/census-instrumentation/opencensus-python/pull/719)) + ## 0.6.0 Released 2019-05-31 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd90e5206..9afc6be1a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,6 +37,7 @@ $ git remote add fork https://github.com/YOUR_GITHUB_USERNAME/opencensus-python. Run tests: ```sh +# Make sure you have all supported versions of Python installed $ pip install nox # Only first time. $ nox ``` diff --git a/README.rst b/README.rst index 377357978..3430c557d 100644 --- a/README.rst +++ b/README.rst @@ -1,16 +1,42 @@ + **Warning** + + OpenCensus and OpenTracing have merged to form + `OpenTelemetry `__, which serves as the + next major version of OpenCensus and OpenTracing. + + OpenTelemetry has now reached feature parity with OpenCensus, with + tracing and metrics SDKs available in .NET, Golang, Java, NodeJS, and + Python. **All OpenCensus Github repositories will be archived**. We + encourage users to migrate to OpenTelemetry. + + To help you gradually migrate your instrumentation to OpenTelemetry, + bridges are available in Java, Go, Python (tracing only), and JS. `Read the + full blog post to learn more + `__. + OpenCensus - A stats collection and distributed tracing framework ================================================================= |gitter| +|travisci| |circleci| |pypi| +|compat_check_pypi| +|compat_check_github| + +.. |travisci| image:: https://travis-ci.org/census-instrumentation/opencensus-python.svg?branch=master + :target: https://travis-ci.org/census-instrumentation/opencensus-python .. |circleci| image:: https://circleci.com/gh/census-instrumentation/opencensus-python.svg?style=shield :target: https://circleci.com/gh/census-instrumentation/opencensus-python .. |gitter| image:: https://badges.gitter.im/census-instrumentation/lobby.svg :target: https://gitter.im/census-instrumentation/lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge .. |pypi| image:: https://badge.fury.io/py/opencensus.svg :target: https://pypi.org/project/opencensus/ +.. |compat_check_pypi| image:: https://python-compatibility-tools.appspot.com/one_badge_image?package=opencensus + :target: https://python-compatibility-tools.appspot.com/one_badge_target?package=opencensus +.. |compat_check_github| image:: https://python-compatibility-tools.appspot.com/one_badge_image?package=git%2Bgit%3A//github.com/census-instrumentation/opencensus-python.git + :target: https://python-compatibility-tools.appspot.com/one_badge_target?package=git%2Bgit%3A//github.com/census-instrumentation/opencensus-python.git `OpenCensus`_ for Python. OpenCensus provides a framework to measure a server's resource usage and collect performance stats. This repository @@ -74,13 +100,13 @@ You can collect traces using the ``Tracer`` `context manager`_: tracer = Tracer(sampler=AlwaysOnSampler()) # Example for creating nested spans - with tracer.span(name='span1') as span1: + with tracer.span(name='span1'): do_something_to_trace() - with span1.span(name='span1_child1') as span1_child1: + with tracer.span(name='span1_child1'): do_something_to_trace() - with span1.span(name='span1_child2') as span1_child2: + with tracer.span(name='span1_child2'): do_something_to_trace() - with tracer.span(name='span2') as span2: + with tracer.span(name='span2'): do_something_to_trace() OpenCensus will collect everything within the ``with`` statement as a single span. @@ -108,9 +134,9 @@ Customization There are several things you can customize in OpenCensus: -* **Blacklist**, which excludes certain hosts and paths from being tracked. +* **Excludelist**, which excludes certain hosts and paths from being tracked. By default, the health check path for the App Engine flexible environment is - not tracked, you can turn it on by excluding it from the blacklist setting. + not tracked, you can turn it on by excluding it from the excludelist setting. * **Exporter**, which sends the traces. By default, the traces are printed to stdout in JSON format. You can choose @@ -164,8 +190,8 @@ information, please read the 'OPENCENSUS': { 'TRACE': { - 'BLACKLIST_HOSTNAMES': ['localhost', '127.0.0.1'], - 'BLACKLIST_PATHS': ['_ah/health'], + 'EXCLUDELIST_HOSTNAMES': ['localhost', '127.0.0.1'], + 'EXCLUDELIST_PATHS': ['_ah/health'], 'SAMPLER': 'opencensus.trace.samplers.ProbabilitySampler(rate=1)', 'EXPORTER': '''opencensus.ext.ocagent.trace_exporter.TraceExporter( service_name='foobar', @@ -189,6 +215,7 @@ OpenCensus supports integration with popular web frameworks, client libraries an - `Google Cloud Client Libraries`_ - `gRPC`_ - `httplib`_ +- `httpx`_ - `logging`_ - `MySQL`_ - `PostgreSQL`_ @@ -199,14 +226,15 @@ OpenCensus supports integration with popular web frameworks, client libraries an - `SQLAlchemy`_ - `threading`_ -Trace Exporter --------------- +Log Exporter +------------ + +- `Azure`_ + +Metrics Exporter +---------------- - `Azure`_ -- `Jaeger`_ -- `OCAgent`_ -- `Stackdriver`_ -- `Zipkin`_ Stats Exporter -------------- @@ -215,13 +243,26 @@ Stats Exporter - `Prometheus`_ - `Stackdriver`_ +Trace Exporter +-------------- + +- `Azure`_ +- `Datadog`_ +- `Jaeger`_ +- `OCAgent`_ +- `Stackdriver`_ +- `Zipkin`_ + .. _Azure: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-azure +.. _Datadog: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-datadog .. _Django: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-django .. _Flask: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-flask +.. _FastAPI: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-fastapi .. _gevent: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-gevent .. _Google Cloud Client Libraries: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-google-cloud-clientlibs .. _gRPC: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-grpc .. _httplib: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-httplib +.. _httpx: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-httpx .. _Jaeger: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-jaeger .. _logging: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-logging .. _MySQL: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-mysql @@ -237,11 +278,6 @@ Stats Exporter .. _threading: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-threading .. _Zipkin: https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-zipkin -Log Exporter --------------- - -- `Azure`_ - ------------ Versioning ------------ diff --git a/context/opencensus-context/CHANGELOG.md b/context/opencensus-context/CHANGELOG.md index a470e2320..d33b890a6 100644 --- a/context/opencensus-context/CHANGELOG.md +++ b/context/opencensus-context/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +## 0.1.3 +Released 2022-08-03 + +- Move `version.py` file into `runtime_context` folder +([#1143](https://github.com/census-instrumentation/opencensus-python/pull/1143)) + +## 0.1.2 +Released 2020-06-29 + +- Release source distribution + ## 0.1.1 Released 2019-05-31 diff --git a/context/opencensus-context/examples/async_span.py b/context/opencensus-context/examples/async_span.py index a91460689..3abcc0db2 100644 --- a/context/opencensus-context/examples/async_span.py +++ b/context/opencensus-context/examples/async_span.py @@ -13,6 +13,7 @@ # limitations under the License. import asyncio + from opencensus.common.runtime_context import RuntimeContext RuntimeContext.register_slot('current_span', None) diff --git a/context/opencensus-context/examples/explicit_threading.py b/context/opencensus-context/examples/explicit_threading.py index ad6af8841..aad6a48c3 100644 --- a/context/opencensus-context/examples/explicit_threading.py +++ b/context/opencensus-context/examples/explicit_threading.py @@ -13,6 +13,7 @@ # limitations under the License. from threading import Thread + from opencensus.common.runtime_context import RuntimeContext RuntimeContext.register_slot('operation_id', '') diff --git a/context/opencensus-context/examples/thread_pool.py b/context/opencensus-context/examples/thread_pool.py index c36d5a304..0f06ea785 100644 --- a/context/opencensus-context/examples/thread_pool.py +++ b/context/opencensus-context/examples/thread_pool.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from multiprocessing.dummy import Pool as ThreadPool -import time import threading +import time +from multiprocessing.dummy import Pool as ThreadPool + from opencensus.common.runtime_context import RuntimeContext RuntimeContext.register_slot('operation_id', '') diff --git a/context/opencensus-context/version.py b/context/opencensus-context/opencensus/common/runtime_context/version.py similarity index 100% rename from context/opencensus-context/version.py rename to context/opencensus-context/opencensus/common/runtime_context/version.py diff --git a/context/opencensus-context/setup.py b/context/opencensus-context/setup.py index 1397ed4fe..cba191aa0 100644 --- a/context/opencensus-context/setup.py +++ b/context/opencensus-context/setup.py @@ -12,13 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup -from version import __version__ +import os + +from setuptools import find_packages, setup + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, "opencensus", "common", "runtime_context", "version.py" +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) setup( name='opencensus-context', - version=__version__, # noqa + version=PACKAGE_INFO["__version__"], # noqa author='OpenCensus Authors', author_email='census-developers@googlegroups.com', classifiers=[ @@ -34,6 +42,8 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Runtime Context', include_package_data=True, diff --git a/context/opencensus-context/tests/test_runtime_context.py b/context/opencensus-context/tests/test_runtime_context.py index 795a165a5..2f26d7b21 100644 --- a/context/opencensus-context/tests/test_runtime_context.py +++ b/context/opencensus-context/tests/test_runtime_context.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest + from opencensus.common.runtime_context import RuntimeContext diff --git a/contrib/opencensus-correlation/CHANGELOG.md b/contrib/opencensus-correlation/CHANGELOG.md index 805c3d79d..cb718632e 100644 --- a/contrib/opencensus-correlation/CHANGELOG.md +++ b/contrib/opencensus-correlation/CHANGELOG.md @@ -2,4 +2,7 @@ ## Unreleased +## 0.3.0 +Released 2019-05-31 + - Add this changelog. diff --git a/contrib/opencensus-correlation/opencensus/common/correlationcontext/__init__.py b/contrib/opencensus-correlation/opencensus/common/correlationcontext/__init__.py index 145a780e0..053ce3981 100644 --- a/contrib/opencensus-correlation/opencensus/common/correlationcontext/__init__.py +++ b/contrib/opencensus-correlation/opencensus/common/correlationcontext/__init__.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opencensus.common.correlationcontext.correlationcontext \ - import CorrelationContext - +from opencensus.common.correlationcontext.correlationcontext import ( + CorrelationContext, +) __all__ = ['CorrelationContext'] diff --git a/contrib/opencensus-correlation/setup.py b/contrib/opencensus-correlation/setup.py index 4f9271017..32ebeb21e 100644 --- a/contrib/opencensus-correlation/setup.py +++ b/contrib/opencensus-correlation/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,6 +34,8 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='W3C Correlation Context', include_package_data=True, diff --git a/contrib/opencensus-correlation/tests/test_correlation_context.py b/contrib/opencensus-correlation/tests/test_correlation_context.py index 98864ba1d..aa0eb0eb8 100644 --- a/contrib/opencensus-correlation/tests/test_correlation_context.py +++ b/contrib/opencensus-correlation/tests/test_correlation_context.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest + from opencensus.common.correlationcontext import CorrelationContext diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index e8241ac9c..3ece097d9 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -2,8 +2,279 @@ ## Unreleased +## 1.1.15 + +Released 2025-06-03 + +- Switch ordering for Statsbeat Attach detection to prioritize Azure Functions + ([#1251](https://github.com/census-instrumentation/opencensus-python/pull/1251)) + +## 1.1.14 + +Released 2025-01-06 + +- Remove status code `206` from retry code + only count batch level for statsbeat +([#1247](https://github.com/census-instrumentation/opencensus-python/pull/1247)) + +## 1.1.13 + +Released 2024-01-03 + +- Changed bit-mapping for `httpx` and `fastapi` integrations +([#1239](https://github.com/census-instrumentation/opencensus-python/pull/1239)) + +## 1.1.12 + +Released 2023-11-28 + +- Fix missing/None fields in `ExceptionDetails` +([#1232](https://github.com/census-instrumentation/opencensus-python/pull/1232)) +- Fix missing/None typeName field in `ExceptionDetails` +([#1234](https://github.com/census-instrumentation/opencensus-python/pull/1234)) + +## 1.1.11 + +Released 2023-10-12 + +- Add str fallback to envelope serialization +([#1196](https://github.com/census-instrumentation/opencensus-python/pull/1196)) +- Remove outerId from exceptiondata +([#1221](https://github.com/census-instrumentation/opencensus-python/pull/1221)) + +## 1.1.10 + +Released 2023-09-18 + +- Add str fallback to envelope serialization +([#1196](https://github.com/census-instrumentation/opencensus-python/pull/1196)) + +## 1.1.9 + +Released 2023-03-10 + +- Fix export of exception information in traces +([#1187](https://github.com/census-instrumentation/opencensus-python/pull/1187)) +- Modify metrics exporter to include setting export interval to 60s +([#1193](https://github.com/census-instrumentation/opencensus-python/pull/1193)) + +## 1.1.8 + +Released 2023-01-18 + +- Disable storage for statsbeat if storage is disabled for exporter +([#1155](https://github.com/census-instrumentation/opencensus-python/pull/1155)) +- Add UK to eu statsbeats +([#1181](https://github.com/census-instrumentation/opencensus-python/pull/1181)) + +## 1.1.7 + +Released 2022-08-18 + +- Add storage existence checks to storing and transmitting in exporter +([#1150](https://github.com/census-instrumentation/opencensus-python/pull/1150)) +- Add 502 and 504 status codes as retriable +([#1153](https://github.com/census-instrumentation/opencensus-python/pull/1153)) +- Fix statsbeat bug - exporting zero values for network statsbeat +([#1155](https://github.com/census-instrumentation/opencensus-python/pull/1155)) + +## 1.1.6 + +Released 2022-08-03 + +- Add statusCode and exceptionType to network statsbeat +([#1138](https://github.com/census-instrumentation/opencensus-python/pull/1138)) + +## 1.1.5 + +Released 2022-07-05 + +- Allow specifying metrics (custom_measurements) for Azure custom events +([#1117](https://github.com/census-instrumentation/opencensus-python/pull/1117)) +- Shutdown Statsbeat when hitting error/exception threshold +([#1127](https://github.com/census-instrumentation/opencensus-python/pull/1127)) +- Fix failure counting statsbeat - refactor status code logic in transport +([#1132](https://github.com/census-instrumentation/opencensus-python/pull/1132)) +- Use logging handler close instead of custom atexit hook +([#1134](https://github.com/census-instrumentation/opencensus-python/pull/1134)) + +## 1.1.4 + +Released 2022-04-20 + +- Statsbeat bug fixes - status codes +([#1113](https://github.com/census-instrumentation/opencensus-python/pull/1113)) +- Statsbeat bug fixes - do not log if statsbeat +([#1116](https://github.com/census-instrumentation/opencensus-python/pull/1116)) +- Add deprecation warning for explicitly using instrumentation key +([#1118](https://github.com/census-instrumentation/opencensus-python/pull/1118)) + +## 1.1.3 + +Released 2022-03-03 + +- Hotfix for version number +([#1108](https://github.com/census-instrumentation/opencensus-python/pull/1108)) + +## 1.1.2 + +Released 2022-03-03 + +- Statsbeat bug fixes, shorten host in network stats +([#1100](https://github.com/census-instrumentation/opencensus-python/pull/1100)) +- Support statsbeat in EU regions +([#1105](https://github.com/census-instrumentation/opencensus-python/pull/1105)) + +## 1.1.1 + +Released 2022-01-19 + +- Fix statsbeats metric names +([#1089](https://github.com/census-instrumentation/opencensus-python/pull/1089)) +- Add AAD statsbeat feature, fix incorrect counting of retry +([#1093](https://github.com/census-instrumentation/opencensus-python/pull/1093)) + +## 1.1.0 + +Released 2021-10-05 + +- Enable AAD authorization via TokenCredential +([#1021](https://github.com/census-instrumentation/opencensus-python/pull/1021)) +- Implement attach rate metrics via Statsbeat +([#1053](https://github.com/census-instrumentation/opencensus-python/pull/1053)) +- Implement network metrics via Statsbeat - Success count +([#1059](https://github.com/census-instrumentation/opencensus-python/pull/1059)) +- Implement network metrics via Statsbeat - Others +([#1062](https://github.com/census-instrumentation/opencensus-python/pull/1062)) +- Implement feature and instrumentation metrics via Statsbeat +([#1076](https://github.com/census-instrumentation/opencensus-python/pull/1076)) +- Support stamp specific redirect in exporters +([#1078](https://github.com/census-instrumentation/opencensus-python/pull/1078)) + +## 1.0.8 + +Released 2021-05-13 + +- Fix `logger.exception` with no exception info throwing error +([#1006](https://github.com/census-instrumentation/opencensus-python/pull/1006)) +- Add `enable_local_storage` to turn on/off local storage + retry + flushing logic +([#1016](https://github.com/census-instrumentation/opencensus-python/pull/1016)) + +## 1.0.7 + +Released 2021-01-25 + +- Hotfix +([#1004](https://github.com/census-instrumentation/opencensus-python/pull/1004)) + +## 1.0.6 + +Released 2021-01-14 + +- Disable heartbeat metrics in exporters + ([#984](https://github.com/census-instrumentation/opencensus-python/pull/984)) +- Loosen instrumentation key validation to GUID + ([#986](https://github.com/census-instrumentation/opencensus-python/pull/986)) + +## 1.0.5 + +Released 2020-10-13 + +- Attach rate metrics via Heartbeat for Web and Function apps + ([#930](https://github.com/census-instrumentation/opencensus-python/pull/930)) +- Attach rate metrics for VM + ([#935](https://github.com/census-instrumentation/opencensus-python/pull/935)) +- Add links in properties for trace exporter envelopes + ([#936](https://github.com/census-instrumentation/opencensus-python/pull/936)) +- Fix attach rate metrics for VM to only ping data service on retry + ([#946](https://github.com/census-instrumentation/opencensus-python/pull/946)) +- Added queue capacity configuration for exporters + ([#949](https://github.com/census-instrumentation/opencensus-python/pull/949)) + +## 1.0.4 + +Released 2020-06-29 + +- Remove dependency rate from standard metrics + ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903)) +- Implement customEvents using AzureEventHandler + ([#925](https://github.com/census-instrumentation/opencensus-python/pull/925)) + +## 1.0.3 + +Released 2020-06-17 + +- Change default path of local storage + ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903)) +- Add support to initialize azure exporters with proxies + ([#902](https://github.com/census-instrumentation/opencensus-python/pull/902)) + +## 1.0.2 + +Released 2020-02-04 + +- Add local storage and retry logic for Azure Metrics Exporter + ([#845](https://github.com/census-instrumentation/opencensus-python/pull/845)) +- Add Fixed-rate sampling logic for Azure Log Exporter + ([#848](https://github.com/census-instrumentation/opencensus-python/pull/848)) +- Implement TelemetryProcessors for Azure exporters + ([#851](https://github.com/census-instrumentation/opencensus-python/pull/851)) + +## 1.0.1 + +Released 2019-11-26 + +- Validate instrumentation key in Azure Exporters + ([#789](https://github.com/census-instrumentation/opencensus-python/pull/789)) +- Add optional custom properties to logging messages + ([#822](https://github.com/census-instrumentation/opencensus-python/pull/822)) + +## 1.0.0 + +Released 2019-09-30 + +- Standard Metrics - Incoming requests execution time + ([#773](https://github.com/census-instrumentation/opencensus-python/pull/773)) +- Implement connection strings + ([#767](https://github.com/census-instrumentation/opencensus-python/pull/767)) + +## 0.7.1 + +Released 2019-08-26 + +- Standard metrics incoming requests per second + ([#758](https://github.com/census-instrumentation/opencensus-python/pull/758)) + +## 0.7.0 + +Released 2019-07-31 + +- Added standard metrics + ([#708](https://github.com/census-instrumentation/opencensus-python/pull/708), + [#718](https://github.com/census-instrumentation/opencensus-python/pull/718), + [#720](https://github.com/census-instrumentation/opencensus-python/pull/720), + [#722](https://github.com/census-instrumentation/opencensus-python/pull/722), + [#724](https://github.com/census-instrumentation/opencensus-python/pull/724)) +- Supported server performance breakdown by operation name + ([#735](https://github.com/census-instrumentation/opencensus-python/pull/735)) + +## 0.3.1 + +Released 2019-06-30 + +- Added metrics exporter + ([#678](https://github.com/census-instrumentation/opencensus-python/pull/678)) + +## 0.2.1 + +Released 2019-06-13 + +- Support span attributes + ([#682](https://github.com/census-instrumentation/opencensus-python/pull/682)) + ## 0.2.0 + Released 2019-05-31 + - Added log exporter ([#657](https://github.com/census-instrumentation/opencensus-python/pull/657), [#668](https://github.com/census-instrumentation/opencensus-python/pull/668)) @@ -13,6 +284,7 @@ Released 2019-05-31 ([#632](https://github.com/census-instrumentation/opencensus-python/pull/632)) ## 0.1.0 + Released 2019-04-24 - Initial release diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 71bc8570c..71fb73127 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -1,5 +1,8 @@ OpenCensus Azure Monitor Exporters -============================================================================ +================================== + +OpenCensus Azure Monitor exporters are on the path to deprecation. They will be officially unsupported by September 2024. Please migrate to the `Azure Monitor OpenTelemetry Distro `_ for the recommended "one-stop-shop" solution or the `Azure Monitor OpenTelemetry exporters `_ for the more hand-on, configurable solution based on `OpenTelemetry `_. +Check out the `migration guide `_ on how to easily migrate Python code. |pypi| @@ -13,9 +16,275 @@ Installation pip install opencensus-ext-azure +Prerequisites +------------- + +* Create an Azure Monitor resource and get the instrumentation key, more information can be found in the official `docs `_. +* Place your instrumentation key in a `connection string` and directly into your code. +* Alternatively, you can specify your `connection string` in an environment variable ``APPLICATIONINSIGHTS_CONNECTION_STRING``. + Usage ----- +Log +~~~ + +The **Azure Monitor Log Handler** allows you to export Python logs to `Azure Monitor`_. + +This example shows how to send a warning level log to Azure Monitor. + +.. code:: python + + import logging + + from opencensus.ext.azure.log_exporter import AzureLogHandler + + logger = logging.getLogger(__name__) + logger.addHandler(AzureLogHandler(connection_string='InstrumentationKey=')) + logger.warning('Hello, World!') + +Correlation +########### + +You can enrich the logs with trace IDs and span IDs by using the `logging integration <../opencensus-ext-logging>`_. + +* Install the `logging integration package <../opencensus-ext-logging>`_ using ``pip install opencensus-ext-logging``. + +.. code:: python + + import logging + + from opencensus.ext.azure.log_exporter import AzureLogHandler + from opencensus.ext.azure.trace_exporter import AzureExporter + from opencensus.trace import config_integration + from opencensus.trace.samplers import ProbabilitySampler + from opencensus.trace.tracer import Tracer + + config_integration.trace_integrations(['logging']) + + logger = logging.getLogger(__name__) + + handler = AzureLogHandler(connection_string='InstrumentationKey=') + handler.setFormatter(logging.Formatter('%(traceId)s %(spanId)s %(message)s')) + logger.addHandler(handler) + + tracer = Tracer( + exporter=AzureExporter(connection_string='InstrumentationKey='), + sampler=ProbabilitySampler(1.0) + ) + + logger.warning('Before the span') + with tracer.span(name='test'): + logger.warning('In the span') + logger.warning('After the span') + +Custom Properties +################# + +You can also add custom properties to your log messages in the *extra* keyword argument using the custom_dimensions field. + +WARNING: For this feature to work, you need to pass a dictionary to the custom_dimensions field. If you pass arguments of any other type, the logger will ignore them. + +.. code:: python + + import logging + + from opencensus.ext.azure.log_exporter import AzureLogHandler + + logger = logging.getLogger(__name__) + logger.addHandler(AzureLogHandler(connection_string='InstrumentationKey=')) + + properties = {'custom_dimensions': {'key_1': 'value_1', 'key_2': 'value_2'}} + logger.warning('action', extra=properties) + +Modifying Logs +############## + +* You can pass a callback function to the exporter to process telemetry before it is exported. +* Your callback function can return `False` if you do not want this envelope exported. +* Your callback function must accept an `envelope `_ data type as its parameter. +* You can see the schema for Azure Monitor data types in the envelopes `here `_. +* The `AzureLogHandler` handles `ExceptionData` and `MessageData` data types. + +.. code:: python + + import logging + + from opencensus.ext.azure.log_exporter import AzureLogHandler + + logger = logging.getLogger(__name__) + + # Callback function to append '_hello' to each log message telemetry + def callback_function(envelope): + envelope.data.baseData.message += '_hello' + return True + + handler = AzureLogHandler(connection_string='InstrumentationKey=') + handler.add_telemetry_processor(callback_function) + logger.addHandler(handler) + logger.warning('Hello, World!') + +Events +###### + +You can send `customEvent` telemetry in exactly the same way you would send `trace` telemetry except using the `AzureEventHandler` instead. + +.. code:: python + + import logging + + from opencensus.ext.azure.log_exporter import AzureEventHandler + + logger = logging.getLogger(__name__) + logger.addHandler(AzureEventHandler(connection_string='InstrumentationKey=')) + logger.setLevel(logging.INFO) + logger.info('Hello, World!') + +Metrics +~~~~~~~ + +The **Azure Monitor Metrics Exporter** allows you to export metrics to `Azure Monitor`_. + +.. code:: python + + from opencensus.ext.azure import metrics_exporter + from opencensus.stats import aggregation as aggregation_module + from opencensus.stats import measure as measure_module + from opencensus.stats import stats as stats_module + from opencensus.stats import view as view_module + from opencensus.tags import tag_map as tag_map_module + + stats = stats_module.stats + view_manager = stats.view_manager + stats_recorder = stats.stats_recorder + + CARROTS_MEASURE = measure_module.MeasureInt("carrots", + "number of carrots", + "carrots") + CARROTS_VIEW = view_module.View("carrots_view", + "number of carrots", + [], + CARROTS_MEASURE, + aggregation_module.CountAggregation()) + + def main(): + # Enable metrics + # Set the interval in seconds to 60s, which is the time interval application insights + # aggregates your metrics + exporter = metrics_exporter.new_metrics_exporter( + connection_string='InstrumentationKey=' + ) + view_manager.register_exporter(exporter) + + view_manager.register_view(CARROTS_VIEW) + mmap = stats_recorder.new_measurement_map() + tmap = tag_map_module.TagMap() + + mmap.measure_int_put(CARROTS_MEASURE, 1000) + mmap.record(tmap) + + print("Done recording metrics") + + if __name__ == "__main__": + main() + +Performance counters +#################### + +The exporter also includes a set of performance counters that are exported to Azure Monitor by default. + +.. code:: python + + import psutil + import time + + from opencensus.ext.azure import metrics_exporter + + def main(): + # Performance counters are sent by default. You can disable performance counters by + # passing in enable_standard_metrics=False into the constructor of + # new_metrics_exporter() + _exporter = metrics_exporter.new_metrics_exporter( + connection_string='InstrumentationKey=', + export_interval=60, + ) + + for i in range(100): + print(psutil.virtual_memory()) + time.sleep(5) + + print("Done recording metrics") + + if __name__ == "__main__": + main() + +Below is a list of performance counters that are currently available: + +- Available Memory (bytes) +- CPU Processor Time (percentage) +- Incoming Request Rate (per second) +- Incoming Request Average Execution Time (milliseconds) +- Process CPU Usage (percentage) +- Process Private Bytes (bytes) + +Modifying Metrics +################# + +* You can pass a callback function to the exporter to process telemetry before it is exported. +* Your callback function can return `False` if you do not want this envelope exported. +* Your callback function must accept an `envelope `_ data type as its parameter. +* You can see the schema for Azure Monitor data types in the envelopes `here `_. +* The `MetricsExporter` handles `MetricData` data types. + +.. code:: python + + from opencensus.ext.azure import metrics_exporter + from opencensus.stats import aggregation as aggregation_module + from opencensus.stats import measure as measure_module + from opencensus.stats import stats as stats_module + from opencensus.stats import view as view_module + from opencensus.tags import tag_map as tag_map_module + + stats = stats_module.stats + view_manager = stats.view_manager + stats_recorder = stats.stats_recorder + + CARROTS_MEASURE = measure_module.MeasureInt("carrots", + "number of carrots", + "carrots") + CARROTS_VIEW = view_module.View("carrots_view", + "number of carrots", + [], + CARROTS_MEASURE, + aggregation_module.CountAggregation()) + + # Callback function to only export the metric if value is greater than 0 + def callback_function(envelope): + return envelope.data.baseData.metrics[0].value > 0 + + def main(): + # Enable metrics + # Set the interval in seconds to 60s, which is the time interval application insights + # aggregates your metrics + exporter = metrics_exporter.new_metrics_exporter( + connection_string='InstrumentationKey=', + export_interval=60, + ) + exporter.add_telemetry_processor(callback_function) + view_manager.register_exporter(exporter) + + view_manager.register_view(CARROTS_VIEW) + mmap = stats_recorder.new_measurement_map() + tmap = tag_map_module.TagMap() + + mmap.measure_int_put(CARROTS_MEASURE, 1000) + mmap.record(tmap) + + print("Done recording metrics") + + if __name__ == "__main__": + main() + Trace ~~~~~ @@ -23,25 +292,30 @@ The **Azure Monitor Trace Exporter** allows you to export `OpenCensus`_ traces t This example shows how to send a span "hello" to Azure Monitor. -* Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. -* Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. - -.. code:: python + .. code:: python from opencensus.ext.azure.trace_exporter import AzureExporter from opencensus.trace.samplers import ProbabilitySampler from opencensus.trace.tracer import Tracer - tracer = Tracer(exporter=AzureExporter(), sampler=ProbabilitySampler(1.0)) + tracer = Tracer( + exporter=AzureExporter( + connection_string='InstrumentationKey=' + ), + sampler=ProbabilitySampler(1.0) + ) with tracer.span(name='hello'): print('Hello, World!') -You can also specify the instrumentation key explicitly in the code. +Integrations +############ + +OpenCensus also supports several `integrations `_ which allows OpenCensus to integrate with third party libraries. + +This example shows how to integrate with the `requests `_ library. -* Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. * Install the `requests integration package <../opencensus-ext-requests>`_ using ``pip install opencensus-ext-requests``. -* Put the instrumentation key in the following code. .. code:: python @@ -55,69 +329,56 @@ You can also specify the instrumentation key explicitly in the code. config_integration.trace_integrations(['requests']) tracer = Tracer( exporter=AzureExporter( - # TODO: replace this with your own instrumentation key. - instrumentation_key='00000000-0000-0000-0000-000000000000', + connection_string='InstrumentationKey=', ), sampler=ProbabilitySampler(1.0), ) with tracer.span(name='parent'): response = requests.get(url='https://www.wikipedia.org/wiki/Rabbit') -Log -~~~ - -The **Azure Monitor Log Handler** allows you to export Python logs to `Azure Monitor`_. - -This example shows how to send a warning level log to Azure Monitor. - -* Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. -* Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. - -.. code:: python - - import logging - - from opencensus.ext.azure.log_exporter import AzureLogHandler - - logger = logging.getLogger(__name__) - logger.addHandler(AzureLogHandler()) - logger.warning('Hello, World!') - -You can enrich the logs with trace IDs and span IDs by using the `logging integration <../opencensus-ext-logging>`_. +Modifying Traces +################ -* Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. -* Install the `logging integration package <../opencensus-ext-logging>`_ using ``pip install opencensus-ext-logging``. -* Put the instrumentation key in ``APPINSIGHTS_INSTRUMENTATIONKEY`` environment variable. +* You can pass a callback function to the exporter to process telemetry before it is exported. +* Your callback function can return `False` if you do not want this envelope exported. +* Your callback function must accept an `envelope `_ data type as its parameter. +* You can see the schema for Azure Monitor data types in the envelopes `here `_. +* The `AzureExporter` handles `Data` data types. .. code:: python - import logging + import requests - from opencensus.ext.azure.log_exporter import AzureLogHandler from opencensus.ext.azure.trace_exporter import AzureExporter from opencensus.trace import config_integration from opencensus.trace.samplers import ProbabilitySampler from opencensus.trace.tracer import Tracer - config_integration.trace_integrations(['logging']) + config_integration.trace_integrations(['requests']) - logger = logging.getLogger(__name__) + # Callback function to add os_type: linux to span properties + def callback_function(envelope): + envelope.data.baseData.properties['os_type'] = 'linux' + return True - handler = AzureLogHandler() - handler.setFormatter(logging.Formatter('%(traceId)s %(spanId)s %(message)s')) - logger.addHandler(handler) - - tracer = Tracer(exporter=AzureExporter(), sampler=ProbabilitySampler(1.0)) + exporter = AzureExporter( + connection_string='InstrumentationKey=' + ) + exporter.add_telemetry_processor(callback_function) + tracer = Tracer(exporter=exporter, sampler=ProbabilitySampler(1.0)) + with tracer.span(name='parent'): + response = requests.get(url='https://www.wikipedia.org/wiki/Rabbit') + +Integrate with Azure Functions +############################## - logger.warning('Before the span') - with tracer.span(name='test'): - logger.warning('In the span') - logger.warning('After the span') +Users who want to capture custom telemetry in Azure Functions environments are encouraged to used the OpenCensus Python Azure Functions `extension `_. More details can be found in this `document `_. References ---------- * `Azure Monitor `_ +* `Official Microsoft Docs `_ * `Examples `_ * `OpenCensus Project `_ diff --git a/contrib/opencensus-ext-azure/examples/logs/correlated.py b/contrib/opencensus-ext-azure/examples/logs/correlated.py index 205a46bc1..9854b9a91 100644 --- a/contrib/opencensus-ext-azure/examples/logs/correlated.py +++ b/contrib/opencensus-ext-azure/examples/logs/correlated.py @@ -24,8 +24,9 @@ logger = logging.getLogger(__name__) -# TODO: you need to specify the instrumentation key in the -# APPINSIGHTS_INSTRUMENTATIONKEY environment variable. +# TODO: you need to specify the instrumentation key in a connection string +# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING +# environment variable. handler = AzureLogHandler() logger.addHandler(handler) @@ -35,3 +36,5 @@ with tracer.span(name='test'): logger.warning('In the span') logger.warning('After the span') + +input("...") diff --git a/contrib/opencensus-ext-azure/examples/logs/error.py b/contrib/opencensus-ext-azure/examples/logs/error.py index 4f801342a..772861cf1 100644 --- a/contrib/opencensus-ext-azure/examples/logs/error.py +++ b/contrib/opencensus-ext-azure/examples/logs/error.py @@ -17,8 +17,9 @@ from opencensus.ext.azure.log_exporter import AzureLogHandler logger = logging.getLogger(__name__) -# TODO: you need to specify the instrumentation key in the -# APPINSIGHTS_INSTRUMENTATIONKEY environment variable. +# TODO: you need to specify the instrumentation key in a connection string +# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING +# environment variable. logger.addHandler(AzureLogHandler()) diff --git a/contrib/opencensus-ext-azure/examples/logs/event.py b/contrib/opencensus-ext-azure/examples/logs/event.py new file mode 100644 index 000000000..9785c5caa --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/logs/event.py @@ -0,0 +1,25 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from opencensus.ext.azure.log_exporter import AzureEventHandler + +logger = logging.getLogger(__name__) +# TODO: you need to specify the instrumentation key in a connection string +# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING +# environment variable. +logger.addHandler(AzureEventHandler()) +logger.setLevel(logging.INFO) +logger.info('Hello, World!') diff --git a/contrib/opencensus-ext-azure/examples/logs/properties.py b/contrib/opencensus-ext-azure/examples/logs/properties.py new file mode 100644 index 000000000..5cfdd3568 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/logs/properties.py @@ -0,0 +1,34 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from opencensus.ext.azure.log_exporter import AzureLogHandler + +logger = logging.getLogger(__name__) +# TODO: you need to specify the instrumentation key in a connection string +# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING +# environment variable. +logger.addHandler(AzureLogHandler()) + +properties = {'custom_dimensions': {'key_1': 'value_1', 'key_2': 'value_2'}} + +# Use properties in logging statements +logger.warning('action', extra=properties) + +# Use properties in exception logs +try: + result = 1 / 0 # generate a ZeroDivisionError +except Exception: + logger.exception('Captured an exception.', extra=properties) diff --git a/contrib/opencensus-ext-azure/examples/logs/simple.py b/contrib/opencensus-ext-azure/examples/logs/simple.py index cdda2b688..2978033b1 100644 --- a/contrib/opencensus-ext-azure/examples/logs/simple.py +++ b/contrib/opencensus-ext-azure/examples/logs/simple.py @@ -17,7 +17,9 @@ from opencensus.ext.azure.log_exporter import AzureLogHandler logger = logging.getLogger(__name__) -# TODO: you need to specify the instrumentation key in the -# APPINSIGHTS_INSTRUMENTATIONKEY environment variable. +# TODO: you need to specify the instrumentation key in a connection string +# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING +# environment variable. logger.addHandler(AzureLogHandler()) -logger.warning('Hello, World!') + +logger.warning("Hello World!") diff --git a/contrib/opencensus-ext-azure/examples/metrics/simple.py b/contrib/opencensus-ext-azure/examples/metrics/simple.py new file mode 100644 index 000000000..353843486 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/metrics/simple.py @@ -0,0 +1,58 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from opencensus.ext.azure import metrics_exporter +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import measure as measure_module +from opencensus.stats import stats as stats_module +from opencensus.stats import view as view_module +from opencensus.tags import tag_map as tag_map_module + +stats = stats_module.stats +view_manager = stats.view_manager +stats_recorder = stats.stats_recorder + +CARROTS_MEASURE = measure_module.MeasureInt("carrots", + "number of carrots", + "carrots") +CARROTS_VIEW = view_module.View("carrots_view", + "number of carrots", + [], + CARROTS_MEASURE, + aggregation_module.CountAggregation()) + + +def main(): + # Enable metrics. Set the interval in seconds to 60s, which is the time + # interval application insights aggregates your metrics + exporter = metrics_exporter.new_metrics_exporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"], + export_interval=60, + ) + view_manager.register_exporter(exporter) + + view_manager.register_view(CARROTS_VIEW) + mmap = stats_recorder.new_measurement_map() + tmap = tag_map_module.TagMap() + + mmap.measure_int_put(CARROTS_MEASURE, 1000) + mmap.record(tmap) + + print("Done recording metrics") + + +if __name__ == "__main__": + main() diff --git a/contrib/opencensus-ext-azure/examples/metrics/sum.py b/contrib/opencensus-ext-azure/examples/metrics/sum.py new file mode 100644 index 000000000..27db12949 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/metrics/sum.py @@ -0,0 +1,63 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from opencensus.ext.azure import metrics_exporter +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import measure as measure_module +from opencensus.stats import stats as stats_module +from opencensus.stats import view as view_module +from opencensus.tags import tag_map as tag_map_module + +stats = stats_module.stats +view_manager = stats.view_manager +stats_recorder = stats.stats_recorder + +REQUEST_MEASURE = measure_module.MeasureFloat("Requests", + "number of requests", + "requests") +NUM_REQUESTS_VIEW = view_module.View("Number of Requests", + "number of requests", + ["url"], + REQUEST_MEASURE, + aggregation_module.SumAggregation()) + + +def main(): + # Enable metrics. Set the interval in seconds to 60s, which is the time + # interval application insights aggregates your metrics + exporter = metrics_exporter.new_metrics_exporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"], + export_interval=60, + ) + view_manager.register_exporter(exporter) + + view_manager.register_view(NUM_REQUESTS_VIEW) + mmap = stats_recorder.new_measurement_map() + tmap = tag_map_module.TagMap() + tmap.insert("url", "http://example.com") + + for i in range(100): + print(i) + mmap.measure_int_put(REQUEST_MEASURE, i) + mmap.record(tmap) + time.sleep(1) + + print("Done recording metrics") + + +if __name__ == "__main__": + main() diff --git a/contrib/opencensus-ext-azure/examples/traces/client.py b/contrib/opencensus-ext-azure/examples/traces/client.py index eb8480089..255492df6 100644 --- a/contrib/opencensus-ext-azure/examples/traces/client.py +++ b/contrib/opencensus-ext-azure/examples/traces/client.py @@ -20,11 +20,12 @@ from opencensus.trace.tracer import Tracer config_integration.trace_integrations(['requests']) -# TODO: you need to specify the instrumentation key in the -# APPINSIGHTS_INSTRUMENTATIONKEY environment variable. +# TODO: you need to specify the instrumentation key in a connection string +# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING +# environment variable. tracer = Tracer(exporter=AzureExporter(), sampler=ProbabilitySampler(1.0)) with tracer.span(name='parent'): with tracer.span(name='child'): - response = requests.get(url='http://localhost:8080/') + response = requests.get(url='http://example.com/') print(response.status_code) print(response.text) diff --git a/contrib/opencensus-ext-azure/examples/traces/config.py b/contrib/opencensus-ext-azure/examples/traces/credential.py similarity index 66% rename from contrib/opencensus-ext-azure/examples/traces/config.py rename to contrib/opencensus-ext-azure/examples/traces/credential.py index 9c4c7fd0f..f17cbada1 100644 --- a/contrib/opencensus-ext-azure/examples/traces/config.py +++ b/contrib/opencensus-ext-azure/examples/traces/credential.py @@ -1,4 +1,4 @@ -# Copyright 2019, OpenCensus Authors +# Copyright 2021, OpenCensus Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,17 +11,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from azure.identity import ClientSecretCredential from opencensus.ext.azure.trace_exporter import AzureExporter from opencensus.trace.samplers import ProbabilitySampler from opencensus.trace.tracer import Tracer +tenant_id = "" +client_id = "" +client_secret = "" + +credential = ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret +) + tracer = Tracer( exporter=AzureExporter( - # TODO: replace the all-zero GUID with your instrumentation key. - instrumentation_key='00000000-0000-0000-0000-000000000000', - ), - sampler=ProbabilitySampler(rate=1.0), + credential=credential, connection_string=""), + sampler=ProbabilitySampler(1.0) ) with tracer.span(name='foo'): diff --git a/contrib/opencensus-ext-azure/examples/traces/custom.py b/contrib/opencensus-ext-azure/examples/traces/custom.py index edcb187c5..abc3497db 100644 --- a/contrib/opencensus-ext-azure/examples/traces/custom.py +++ b/contrib/opencensus-ext-azure/examples/traces/custom.py @@ -13,6 +13,7 @@ # limitations under the License. from flask import Flask + from opencensus.ext.flask.flask_middleware import FlaskMiddleware app = Flask(__name__) @@ -21,7 +22,8 @@ 'TRACE': { 'SAMPLER': 'opencensus.trace.samplers.ProbabilitySampler(rate=1.0)', 'EXPORTER': '''opencensus.ext.azure.trace_exporter.AzureExporter( - instrumentation_key='00000000-0000-0000-0000-000000000000', + connection_string= + 'InstrumentationKey=00000000-0000-0000-0000-000000000000', )''', }, } diff --git a/contrib/opencensus-ext-azure/examples/traces/django/logfile b/contrib/opencensus-ext-azure/examples/traces/django/logfile new file mode 100644 index 000000000..c58b7d762 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/logfile @@ -0,0 +1,8 @@ +2022-10-26 15:48:57,700 INFO This is an INFO level log entry. +2022-10-26 15:48:57,700 WARNING This is a WARNING level log entry. +2022-10-26 15:48:57,701 ERROR This is an ERROR level log entry. +2022-10-26 15:48:57,702 CRITICAL This is a CRITICAL level log entry. +2022-10-26 16:10:22,849 INFO This is an INFO level log entry. +2022-10-26 16:10:22,850 WARNING This is a WARNING level log entry. +2022-10-26 16:10:22,850 ERROR This is an ERROR level log entry. +2022-10-26 16:10:22,850 CRITICAL This is a CRITICAL level log entry. diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/db.sqlite3 b/contrib/opencensus-ext-azure/examples/traces/django/mysite/db.sqlite3 new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/manage.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/manage.py new file mode 100644 index 000000000..538658793 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/manage.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/__init__.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/__init__.py new file mode 100644 index 000000000..296af4941 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/asgi.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/asgi.py new file mode 100644 index 000000000..4d82b9f8c --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/asgi.py @@ -0,0 +1,29 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +ASGI config for mysite project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +application = get_asgi_application() diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/settings.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/settings.py new file mode 100644 index 000000000..da26367f2 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/settings.py @@ -0,0 +1,184 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Django settings for mysite project. + +Generated by 'django-admin startproject' using Django 3.2.14. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.2/ref/settings/ +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'secret_key_for_test' + +ALLOWED_HOSTS = ['*'] + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'opencensus.ext.django.middleware.OpencensusMiddleware', +] + +MY_CONNECTION_STRING = "''" + +OPENCENSUS = { + 'TRACE': { + 'SAMPLER': 'opencensus.trace.samplers.ProbabilitySampler(rate=1.0)', + 'EXPORTER': 'opencensus.ext.azure.trace_exporter.AzureExporter(connection_string=' + MY_CONNECTION_STRING + ')', # noqa: E501 + } +} + +ROOT_URLCONF = 'mysite.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'mysite.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa: E501 + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa: E501 + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa: E501 + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa: E501 + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ + +STATIC_URL = '/static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False +ALLOWED_HOSTS = ["*"] + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'timestamp': { + 'format': '{asctime} {levelname} {message}', + 'style': '{', + }, + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'timestamp', + }, + 'logfile': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'formatter': 'timestamp', + 'filename': str(BASE_DIR) + "/../logfile", + }, + 'azure': { + 'level': "DEBUG", + 'class': "opencensus.ext.azure.log_exporter.AzureLogHandler", + 'connection_string': MY_CONNECTION_STRING, + 'formatter': 'timestamp', + }, + }, + 'loggers': { + 'custom': { + 'level': 'INFO', + 'handlers': ['console', 'logfile', 'azure'] + } + } +} diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/urls.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/urls.py new file mode 100644 index 000000000..407f10f8b --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/urls.py @@ -0,0 +1,35 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""mysite URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import include, path + +urlpatterns = [ + path('admin/', admin.site.urls), + path('', include('polls.urls')), +] diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/wsgi.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/wsgi.py new file mode 100644 index 000000000..247a03089 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/mysite/wsgi.py @@ -0,0 +1,29 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +WSGI config for mysite project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +application = get_wsgi_application() diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/__init__.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/__init__.py new file mode 100644 index 000000000..296af4941 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/admin.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/admin.py new file mode 100644 index 000000000..296af4941 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/admin.py @@ -0,0 +1,13 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/apps.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/apps.py new file mode 100644 index 000000000..41e62e385 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/apps.py @@ -0,0 +1,19 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from django.apps import AppConfig + + +class PollsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'polls' diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/migrations/__init__.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/migrations/__init__.py new file mode 100644 index 000000000..296af4941 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/migrations/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/models.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/models.py new file mode 100644 index 000000000..296af4941 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/models.py @@ -0,0 +1,13 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/tests.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/tests.py new file mode 100644 index 000000000..296af4941 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/tests.py @@ -0,0 +1,13 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/urls.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/urls.py new file mode 100644 index 000000000..580d73bef --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/urls.py @@ -0,0 +1,20 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from django.urls import path + +from . import views + +urlpatterns = [ + path('', views.index, name='index'), +] diff --git a/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/views.py b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/views.py new file mode 100644 index 000000000..833f77d5e --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/traces/django/mysite/polls/views.py @@ -0,0 +1,29 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from django.http import HttpResponse + +# Logging configured through settings.LOGGING in settings.py +logger = logging.getLogger('custom') + + +# Distributed tracing configured through settings.OPENCENSUS in settings.py +def index(request): + logger.debug('This is a DEBUG level log entry.') + logger.info('This is an INFO level log entry.') + logger.warning('This is a WARNING level log entry.') + logger.error('This is an ERROR level log entry.') + logger.critical('This is a CRITICAL level log entry.') + return HttpResponse("Hello, world. You're at the polls index.") diff --git a/contrib/opencensus-ext-azure/examples/traces/server.py b/contrib/opencensus-ext-azure/examples/traces/server.py index a76e5d3a2..0b4831d5b 100644 --- a/contrib/opencensus-ext-azure/examples/traces/server.py +++ b/contrib/opencensus-ext-azure/examples/traces/server.py @@ -12,16 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from flask import Flask import requests +from flask import Flask from opencensus.ext.azure.trace_exporter import AzureExporter from opencensus.ext.flask.flask_middleware import FlaskMiddleware from opencensus.trace import config_integration from opencensus.trace.samplers import ProbabilitySampler -# TODO: you need to specify the instrumentation key in the -# APPINSIGHTS_INSTRUMENTATIONKEY environment variable. +# TODO: you need to specify the instrumentation key in a connection string +# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING +# environment variable. app = Flask(__name__) middleware = FlaskMiddleware( app, diff --git a/contrib/opencensus-ext-azure/examples/traces/simple.py b/contrib/opencensus-ext-azure/examples/traces/simple.py index 49c69d6e1..b0008f464 100644 --- a/contrib/opencensus-ext-azure/examples/traces/simple.py +++ b/contrib/opencensus-ext-azure/examples/traces/simple.py @@ -16,8 +16,9 @@ from opencensus.trace.samplers import ProbabilitySampler from opencensus.trace.tracer import Tracer -# TODO: you need to specify the instrumentation key in the -# APPINSIGHTS_INSTRUMENTATIONKEY environment variable. +# TODO: you need to specify the instrumentation key in a connection string +# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING +# environment variable. tracer = Tracer(exporter=AzureExporter(), sampler=ProbabilitySampler(1.0)) with tracer.span(name='foo'): diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py index 113604a94..96aa54ed3 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py @@ -12,29 +12,127 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os -import sys +import tempfile from opencensus.ext.azure.common.protocol import BaseObject +INGESTION_ENDPOINT = 'ingestionendpoint' +INSTRUMENTATION_KEY = 'instrumentationkey' +TEMPDIR_PREFIX = "opencensus-python-" + +_logger = logging.getLogger(__name__) + + +def process_options(options): + # Connection string/ikey + code_cs = parse_connection_string(options.connection_string) + code_ikey = options.instrumentation_key + env_cs = parse_connection_string( + os.getenv('APPLICATIONINSIGHTS_CONNECTION_STRING')) + env_ikey = os.getenv('APPINSIGHTS_INSTRUMENTATIONKEY') + + # Deprecation note about explicit instrumentation key usage + if (not code_cs and code_ikey) or (not env_cs and env_ikey): + _logger.warning( + "DeprecationWarning: Explicitly using instrumentation key is" + "deprecated. Please use a connection string instead." + ) + + # The priority of which value takes on the instrumentation key is: + # 1. Key from explicitly passed in connection string + # 2. Key from explicitly passed in instrumentation key + # 3. Key from connection string in environment variable + # 4. Key from instrumentation key in environment variable + options.instrumentation_key = code_cs.get(INSTRUMENTATION_KEY) \ + or code_ikey \ + or env_cs.get(INSTRUMENTATION_KEY) \ + or env_ikey + # The priority of the ingestion endpoint is as follows: + # 1. The endpoint explicitly passed in connection string + # 2. The endpoint from the connection string in environment variable + # 3. The default breeze endpoint + endpoint = code_cs.get(INGESTION_ENDPOINT) \ + or env_cs.get(INGESTION_ENDPOINT) \ + or 'https://dc.services.visualstudio.com' + options.endpoint = endpoint + + # Authorization + # `azure.core.credentials.TokenCredential` class must be valid + if options.credential and not hasattr(options.credential, 'get_token'): + raise ValueError( + 'Must pass in valid TokenCredential.' + ) + + # storage path + if options.storage_path is None: + TEMPDIR_SUFFIX = options.instrumentation_key or "" + options.storage_path = os.path.join( + tempfile.gettempdir(), + TEMPDIR_PREFIX + TEMPDIR_SUFFIX + ) + + # proxies + if options.proxies is None: + options.proxies = '{}' + + +def parse_connection_string(connection_string): + if connection_string is None: + return {} + try: + pairs = connection_string.split(';') + result = dict(s.split('=') for s in pairs) + # Convert keys to lower-case due to case type-insensitive checking + result = {key.lower(): value for key, value in result.items()} + except Exception: + raise ValueError('Invalid connection string') + # Validate authorization + auth = result.get('authorization') + if auth is not None and auth.lower() != 'ikey': + raise ValueError('Invalid authorization mechanism') + # Construct the ingestion endpoint if not passed in explicitly + if result.get(INGESTION_ENDPOINT) is None: + endpoint_suffix = '' + location_prefix = '' + suffix = result.get('endpointsuffix') + if suffix is not None: + endpoint_suffix = suffix + # Get regional information if provided + prefix = result.get('location') + if prefix is not None: + location_prefix = prefix + '.' + endpoint = 'https://' + location_prefix + 'dc.' + endpoint_suffix + result[INGESTION_ENDPOINT] = endpoint + else: + # Default to None if cannot construct + result[INGESTION_ENDPOINT] = None + return result + class Options(BaseObject): + def __init__(self, *args, **kwargs): + super(Options, self).__init__(*args, **kwargs) + process_options(self) + _default = BaseObject( + connection_string=None, + credential=None, # Credential class used by AAD auth + enable_local_storage=True, + enable_standard_metrics=True, # Used by metrics exporter, True to send standard metrics # noqa: E501 endpoint='https://dc.services.visualstudio.com/v2/track', export_interval=15.0, grace_period=5.0, - instrumentation_key=os.getenv('APPINSIGHTS_INSTRUMENTATIONKEY', None), + instrumentation_key=None, + logging_sampling_rate=1.0, # Used by log exporter, controls sampling max_batch_size=100, minimum_retry_interval=60, # minimum retry interval in seconds - proxy=None, + proxies=None, # string maps url schemes to the url of the proxies + queue_capacity=8192, storage_maintenance_period=60, - storage_max_size=100*1024*1024, - storage_path=os.path.join( - os.path.expanduser('~'), - '.opencensus', - '.azure', - os.path.basename(sys.argv[0]) or '.console', - ), + storage_max_size=50*1024*1024, # 50MiB + storage_path=None, storage_retention_period=7*24*60*60, timeout=10.0, # networking timeout in seconds ) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py index 01faa3bee..a4c0b9df2 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py @@ -16,9 +16,9 @@ import threading import time -from opencensus.common.schedule import Queue -from opencensus.common.schedule import QueueEvent +from opencensus.common.schedule import Queue, QueueEvent from opencensus.ext.azure.common import Options +from opencensus.trace import execution_context class BaseExporter(object): @@ -28,7 +28,7 @@ def __init__(self, **options): self.max_batch_size = options.max_batch_size # TODO: queue should be moved to tracer # too much refactor work, leave to the next PR - self._queue = Queue(capacity=8192) # TODO: make this configurable + self._queue = Queue(capacity=options.queue_capacity) # TODO: worker should not be created in the base exporter self._worker = Worker(self._queue, self) self._worker.start() @@ -61,9 +61,14 @@ def __init__(self, src, dst): self.src = src self.dst = dst self._stopping = False - super(Worker, self).__init__() + super(Worker, self).__init__( + name="AzureExporter Worker" + ) def run(self): # pragma: NO COVER + # Indicate that this thread is an exporter thread. + # Used to suppress tracking of requests in this thread + execution_context.set_is_exporter(True) src = self.src dst = self.dst while True: diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/processor.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/processor.py new file mode 100644 index 000000000..4a98afa50 --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/processor.py @@ -0,0 +1,63 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +logger = logging.getLogger(__name__) + + +class ProcessorMixin(object): + """ProcessorMixin adds the ability to process telemetry processors + + Telemetry processors are functions that are called before exporting of + telemetry to possibly modify the envelope contents. + """ + + def add_telemetry_processor(self, processor): + """Adds telemetry processor to the collection. Telemetry processors + will be called one by one before telemetry item is pushed for sending + and in the order they were added. + + :param processor: The processor to add. + """ + self._telemetry_processors.append(processor) + + def clear_telemetry_processors(self): + """Removes all telemetry processors""" + self._telemetry_processors = [] + + def apply_telemetry_processors(self, envelopes): + """Applies all telemetry processors in the order they were added. + + This function will return the list of envelopes to be exported after + each processor has been run sequentially. Individual processors can + throw exceptions and fail, but the applying of all telemetry processors + will proceed (not fast fail). Processors also return True if envelope + should be included for exporting, False otherwise. + + :param envelopes: The envelopes to apply each processor to. + """ + filtered_envelopes = [] + for envelope in envelopes: + accepted = True + for processor in self._telemetry_processors: + try: + if processor(envelope) is False: + accepted = False + break + except Exception as ex: + logger.warning('Telemetry processor failed with: %s.', ex) + if accepted: + filtered_envelopes.append(envelope) + return filtered_envelopes diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py index 23ce42f43..64abeda6f 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py @@ -65,6 +65,24 @@ def __init__(self, *args, **kwargs): self.baseType = self.baseType +class DataPoint(BaseObject): + _default = BaseObject( + ns='', + name='', + kind=None, + value=0.0, + count=None, + min=None, + max=None, + stdDev=None, + ) + + def __init__(self, *args, **kwargs): + super(DataPoint, self).__init__(*args, **kwargs) + self.name = self.name + self.value = self.value + + class Envelope(BaseObject): _default = BaseObject( ver=1, @@ -129,6 +147,19 @@ def __init__(self, *args, **kwargs): self.message = self.message +class MetricData(BaseObject): + _default = BaseObject( + ver=2, + metrics=[], + properties=None, + ) + + def __init__(self, *args, **kwargs): + super(MetricData, self).__init__(*args, **kwargs) + self.ver = self.ver + self.metrics = self.metrics + + class RemoteDependency(BaseObject): _default = BaseObject( ver=2, diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py index a7fce53d7..46958f384 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py @@ -1,10 +1,13 @@ import datetime import json -import random +import logging import os +import random from opencensus.common.schedule import PeriodicTask +logger = logging.getLogger(__name__) + def _fmt(timestamp): return timestamp.strftime('%Y-%m-%dT%H%M%S.%f') @@ -22,14 +25,13 @@ class LocalFileBlob(object): def __init__(self, fullpath): self.fullpath = fullpath - def delete(self, silent=False): + def delete(self): try: os.remove(self.fullpath) except Exception: - if not silent: - raise + pass # keep silent - def get(self, silent=False): + def get(self): try: with open(self.fullpath, 'r') as file: return tuple( @@ -37,10 +39,9 @@ def get(self, silent=False): for line in file.readlines() ) except Exception: - if not silent: - raise + pass # keep silent - def put(self, data, lease_period=0, silent=False): + def put(self, data, lease_period=0): try: fullpath = self.fullpath + '.tmp' with open(fullpath, 'w') as file: @@ -56,8 +57,7 @@ def put(self, data, lease_period=0, silent=False): os.rename(fullpath, self.fullpath) return self except Exception: - if not silent: - raise + pass # keep silent def lease(self, period): timestamp = _now() + _seconds(period) @@ -77,21 +77,23 @@ class LocalFileStorage(object): def __init__( self, path, - max_size=100*1024*1024, # 100MB + max_size=50*1024*1024, # 50MiB maintenance_period=60, # 1 minute retention_period=7*24*60*60, # 7 days write_timeout=60, # 1 minute + source=None, ): self.path = os.path.abspath(path) self.max_size = max_size self.maintenance_period = maintenance_period self.retention_period = retention_period self.write_timeout = write_timeout - self._maintenance_routine(silent=False) + # Run maintenance routine once upon instantiating + self._maintenance_routine() self._maintenance_task = PeriodicTask( interval=self.maintenance_period, function=self._maintenance_routine, - kwargs={'silent': True}, + name='{} Storage Worker'.format(source) ) self._maintenance_task.daemon = True self._maintenance_task.start() @@ -106,19 +108,18 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.close() - def _maintenance_routine(self, silent=False): + def _maintenance_routine(self): try: if not os.path.isdir(self.path): os.makedirs(self.path) except Exception: - if not silent: - raise + # Race case will throw OSError which we can ignore + pass try: for blob in self.gets(): pass except Exception: - if not silent: - raise + pass # keep silent def gets(self): now = _now() @@ -132,7 +133,9 @@ def gets(self): if path.endswith('.tmp'): if name < timeout_deadline: try: - os.remove(path) # TODO: log data loss + os.remove(path) + logger.warning( + 'File write exceeded timeout. Dropping telemetry') except Exception: pass # keep silent if path.endswith('.lock'): @@ -147,7 +150,10 @@ def gets(self): if path.endswith('.blob'): if name < retention_deadline: try: - os.remove(path) # TODO: log data loss + os.remove(path) + logger.warning( + 'File write exceeded retention.' + + 'Dropping telemetry') except Exception: pass # keep silent else: @@ -161,7 +167,9 @@ def get(self): pass return None - def put(self, data, lease_period=0, silent=False): + def put(self, data, lease_period=0): + if not self._check_storage_size(): + return None blob = LocalFileBlob(os.path.join( self.path, '{}-{}.blob'.format( @@ -169,4 +177,29 @@ def put(self, data, lease_period=0, silent=False): '{:08x}'.format(random.getrandbits(32)), # thread-safe random ), )) - return blob.put(data, lease_period=lease_period, silent=silent) + return blob.put(data, lease_period=lease_period) + + def _check_storage_size(self): + size = 0 + for dirpath, dirnames, filenames in os.walk(self.path): + for f in filenames: + fp = os.path.join(dirpath, f) + # skip if it is symbolic link + if not os.path.islink(fp): + try: + size += os.path.getsize(fp) + except OSError: + logger.error( + "Path %s does not exist or is inaccessible.", fp + ) + continue + if size >= self.max_size: + logger.warning( + "Persistent storage max capacity has been " + "reached. Currently at %sKB. Telemetry will be " + "lost. Please consider increasing the value of " + "'storage_max_size' in exporter config.", + format(size/1024) + ) + return False + return True diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py index 43b1e1486..cf0cf622a 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py @@ -14,25 +14,78 @@ import json import logging +import threading +import time + import requests +from azure.core.exceptions import ClientAuthenticationError +from azure.identity._exceptions import CredentialUnavailableError + +from opencensus.ext.azure.statsbeat import state + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse -from opencensus.trace import execution_context logger = logging.getLogger(__name__) +_MAX_CONSECUTIVE_REDIRECTS = 10 +_MONITOR_OAUTH_SCOPE = "https://monitor.azure.com//.default" +_requests_lock = threading.Lock() +_requests_map = {} +_REACHED_INGESTION_STATUS_CODES = (200, 206, 402, 408, 429, 439, 500) +REDIRECT_STATUS_CODES = (307, 308) +RETRYABLE_STATUS_CODES = ( + 401, # Unauthorized + 403, # Forbidden + 408, # Request Timeout + 429, # Too Many Requests - retry after + 500, # Internal server error + 502, # Bad Gateway + 503, # Service unavailable + 504, # Gateway timeout +) +THROTTLE_STATUS_CODES = ( + 402, # Quota, too Many Requests over extended time + 439, # Quota, too Many Requests over extended time (legacy) +) + + +class TransportStatusCode: + SUCCESS = 0 + RETRY = 1 + DROP = 2 + STATSBEAT_SHUTDOWN = 3 + + class TransportMixin(object): + + # check to see whether its the case of stats collection + def _check_stats_collection(self): + return state.is_statsbeat_enabled() and \ + not state.get_statsbeat_shutdown() and \ + not self._is_stats_exporter() + + # check if the current exporter is a statsbeat metric exporter + # only applies to metrics exporter + def _is_stats_exporter(self): + return hasattr(self, '_is_stats') and self._is_stats + def _transmit_from_storage(self): - for blob in self.storage.gets(): - # give a few more seconds for blob lease operation - # to reduce the chance of race (for perf consideration) - if blob.lease(self.options.timeout + 5): - envelopes = blob.get() # TODO: handle error - result = self._transmit(envelopes) - if result > 0: - blob.lease(result) - else: - blob.delete(silent=True) + if self.storage: + for blob in self.storage.gets(): + # give a few more seconds for blob lease operation + # to reduce the chance of race (for perf consideration) + if blob.lease(self.options.timeout + 5): + envelopes = blob.get() + result = self._transmit(envelopes) + if result is TransportStatusCode.RETRY: + blob.lease(result) + else: + blob.delete() def _transmit(self, envelopes): """ @@ -42,94 +95,261 @@ def _transmit(self, envelopes): Return the next retry time in seconds for retryable failure. This function should never throw exception. """ - # TODO: prevent requests being tracked - blacklist_hostnames = execution_context.get_opencensus_attr( - 'blacklist_hostnames', - ) - execution_context.set_opencensus_attr( - 'blacklist_hostnames', - ['dc.services.visualstudio.com'], - ) + if not envelopes: + return 0 + status = None + exception = None try: + start_time = time.time() + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json; charset=utf-8', + } + endpoint = self.options.endpoint + if self.options.credential: + token = self.options.credential.get_token(_MONITOR_OAUTH_SCOPE) + headers["Authorization"] = "Bearer {}".format(token.token) + endpoint += '/v2.1/track' + proxies = json.loads(self.options.proxies) + allow_redirects = len(proxies) != 0 + response = requests.post( - url=self.options.endpoint, - data=json.dumps(envelopes), - headers={ - 'Accept': 'application/json', - 'Content-Type': 'application/json; charset=utf-8', - }, + url=endpoint, + data=json.dumps(envelopes, default=str), + headers=headers, timeout=self.options.timeout, + proxies=proxies, + allow_redirects=allow_redirects, ) - except Exception as ex: # TODO: consider RequestException - logger.warning('Transient client side error %s.', ex) - # client side error (retryable) - return self.options.minimum_retry_interval + except requests.Timeout as ex: + if not self._is_stats_exporter(): + logger.warning( + 'Request time out. Ingestion may be backed up. Retrying.') + status = TransportStatusCode.RETRY + exception = ex + except requests.RequestException as ex: + if not self._is_stats_exporter(): + logger.warning( + 'Retrying due to transient client side error %s.', ex) + status = TransportStatusCode.RETRY + exception = ex + except CredentialUnavailableError as ex: + if not self._is_stats_exporter(): + logger.warning('Credential error. %s. Dropping telemetry.', ex) + status = TransportStatusCode.DROP + exception = ex + except ClientAuthenticationError as ex: + if not self._is_stats_exporter(): + logger.warning('Authentication error %s', ex) + status = TransportStatusCode.RETRY + exception = ex + except Exception as ex: + if not self._is_stats_exporter(): + logger.warning( + 'Error when sending request %s. Dropping telemetry.', ex) + status = TransportStatusCode.DROP + exception = ex finally: - execution_context.set_opencensus_attr( - 'blacklist_hostnames', - blacklist_hostnames, - ) + if self._check_stats_collection(): + _update_requests_map('count') + end_time = time.time() + if self._check_stats_collection(): + _update_requests_map('duration', value=end_time-start_time) + + if status is not None and exception is not None: + if self._check_stats_collection(): + _update_requests_map('exception', value=exception.__class__.__name__) # noqa: E501 + return status + if self._is_stats_exporter() and \ + not state.get_statsbeat_shutdown() and \ + not state.get_statsbeat_initial_success(): + # If ingestion threshold during statsbeat initialization is + # reached, return back code to shut it down + if _statsbeat_failure_reached_threshold(): + return TransportStatusCode.STATSBEAT_SHUTDOWN + text = 'N/A' - data = None + status_code = 0 try: text = response.text + status_code = response.status_code except Exception as ex: - logger.warning('Error while reading response body %s.', ex) - else: + if not self._is_stats_exporter(): + logger.warning('Error while reading response body %s.', ex) + if self._check_stats_collection(): + _update_requests_map('exception', value=ex.__class__.__name__) + return TransportStatusCode.DROP + + if self._is_stats_exporter() and \ + not state.get_statsbeat_shutdown() and \ + not state.get_statsbeat_initial_success(): + # If statsbeat exporter, record initialization as success if + # appropriate status code is returned + if _reached_ingestion_status_code(status_code): + state.set_statsbeat_initial_success(True) + elif _statsbeat_failure_reached_threshold(): + # If ingestion threshold during statsbeat initialization is + # reached, return back code to shut it down + return TransportStatusCode.STATSBEAT_SHUTDOWN + + if status_code == 200: # Success + self._consecutive_redirects = 0 + if self._check_stats_collection(): + _update_requests_map('success') + return TransportStatusCode.SUCCESS + elif status_code == 206: # Partial Content + data = None try: data = json.loads(text) - except Exception: - pass - if response.status_code == 200: - logger.info('Transmission succeeded: %s.', text) - return 0 - if response.status_code == 206: # Partial Content - # TODO: store the unsent data + except Exception as ex: + if not self._is_stats_exporter(): + logger.warning('Error while reading response body %s for partial content.', ex) # noqa: E501 + if self._check_stats_collection(): + _update_requests_map('exception', value=ex.__class__.__name__) # noqa: E501 + return TransportStatusCode.DROP if data: try: resend_envelopes = [] for error in data['errors']: - if error['statusCode'] in ( - 429, # Too Many Requests - 500, # Internal Server Error - 503, # Service Unavailable - ): + if _status_code_is_retryable(error['statusCode']): resend_envelopes.append(envelopes[error['index']]) else: - logger.error( - 'Data drop %s: %s %s.', - error['statusCode'], - error['message'], - envelopes[error['index']], - ) - if resend_envelopes: + if not self._is_stats_exporter(): + logger.error( + 'Data drop %s: %s %s.', + error['statusCode'], + error['message'], + envelopes[error['index']], + ) + if self.storage and resend_envelopes: self.storage.put(resend_envelopes) except Exception as ex: + if not self._is_stats_exporter(): + logger.error( + 'Error while processing %s: %s %s.', + status_code, + text, + ex, + ) + if self._check_stats_collection(): + _update_requests_map('exception', value=ex.__class__.__name__) # noqa: E501 + return TransportStatusCode.DROP + # cannot parse response body, fallback to retry + elif _status_code_is_redirect(status_code): # Redirect + # for statsbeat, these are not tracked as success nor failures + self._consecutive_redirects += 1 + if self._consecutive_redirects < _MAX_CONSECUTIVE_REDIRECTS: + if response.headers: + location = response.headers.get("location") + if location: + url = urlparse(location) + if url.scheme and url.netloc: + # Change the host to the new redirected host + self.options.endpoint = "{}://{}".format(url.scheme, url.netloc) # noqa: E501 + # Attempt to export again + return self._transmit(envelopes) + if not self._is_stats_exporter(): + logger.error( + "Error parsing redirect information." + ) + else: + if not self._is_stats_exporter(): logger.error( - 'Error while processing %s: %s %s.', - response.status_code, + "Error sending telemetry because of circular redirects." # noqa: E501 + " Please check the integrity of your connection string." # noqa: E501 + ) + # If redirect but did not return, exception occured + if self._check_stats_collection(): + _update_requests_map('exception', value="Circular Redirect") + return TransportStatusCode.DROP + elif _status_code_is_throttle(status_code): # Throttle + if self._check_stats_collection(): + # 402: Monthly Quota Exceeded (new SDK) + # 439: Monthly Quota Exceeded (old SDK) <- Currently OC SDK + _update_requests_map('throttle', value=status_code) + if not self._is_stats_exporter(): + logger.warning( + 'Telemetry was throttled %s: %s.', + status_code, text, - ex, ) - return -response.status_code - # cannot parse response body, fallback to retry - if response.status_code in ( - 206, # Partial Content - 429, # Too Many Requests - 500, # Internal Server Error - 503, # Service Unavailable - ): - logger.warning( - 'Transient server side error %s: %s.', - response.status_code, - text, - ) - # server side error (retryable) - return self.options.minimum_retry_interval - logger.error( - 'Non-retryable server side error %s: %s.', - response.status_code, - text, - ) - # server side error (non-retryable) - return -response.status_code + return TransportStatusCode.DROP + elif _status_code_is_retryable(status_code): # Retry + if not self._is_stats_exporter(): + if status_code == 401: # Authentication error + logger.warning( + 'Authentication error %s: %s. Retrying.', + status_code, + text, + ) + elif status_code == 403: + # Forbidden error + # Can occur when v2 endpoint is used while AI resource is configured # noqa: E501 + # with disableLocalAuth + logger.warning( + 'Forbidden error %s: %s. Retrying.', + status_code, + text, + ) + else: + logger.warning( + 'Transient server side error %s: %s. Retrying.', + status_code, + text, + ) + if self._check_stats_collection(): + _update_requests_map('retry', value=status_code) + return TransportStatusCode.RETRY + else: + # 400 and 404 will be tracked as failure count + # 400 - Invalid - The server cannot or will not process the request due to the invalid telemetry (invalid data, iKey) # noqa: E501 + # 404 - Ingestion is allowed only from stamp specific endpoint - must update connection string # noqa: E501 + if self._check_stats_collection(): + _update_requests_map('failure', value=status_code) + # Other, server side error (non-retryable) + if not self._is_stats_exporter(): + logger.error( + 'Non-retryable server side error %s: %s.', + status_code, + text, + ) + return TransportStatusCode.DROP + + +def _status_code_is_redirect(status_code): + return status_code in REDIRECT_STATUS_CODES + + +def _status_code_is_throttle(status_code): + return status_code in THROTTLE_STATUS_CODES + + +def _status_code_is_retryable(status_code): + return status_code in RETRYABLE_STATUS_CODES + + +def _reached_ingestion_status_code(status_code): + return status_code in _REACHED_INGESTION_STATUS_CODES + + +def _statsbeat_failure_reached_threshold(): + # increment failure counter for sending statsbeat if in initialization + state.increment_statsbeat_initial_failure_count() + return state.get_statsbeat_initial_failure_count() >= 3 + + +def _update_requests_map(type_name, value=None): + # value is either None, duration, status_code or exc_name + with _requests_lock: + if type_name == "success" or type_name == "count": # success, count + _requests_map[type_name] = _requests_map.get(type_name, 0) + 1 + elif type_name == "duration": # value will be duration + _requests_map[type_name] = _requests_map.get(type_name, 0) + value # noqa: E501 + else: # exception, failure, retry, throttle + # value will be a key (status_code/exc_name) + prev = 0 + if _requests_map.get(type_name): + prev = _requests_map.get(type_name).get(value, 0) + else: + _requests_map[type_name] = {} + _requests_map[type_name][value] = prev + 1 diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/utils.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/utils.py index 97c6148e3..75c839ee1 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/utils.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/utils.py @@ -16,16 +16,11 @@ import locale import os import platform +import re import sys -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - +from opencensus.common.utils import timestamp_to_microseconds, to_iso_str from opencensus.common.version import __version__ as opencensus_version -from opencensus.common.utils import timestamp_to_microseconds -from opencensus.common.utils import to_iso_str from opencensus.ext.azure.common.version import __version__ as ext_version azure_monitor_context = { @@ -63,5 +58,22 @@ def timestamp_to_iso_str(timestamp): return to_iso_str(datetime.datetime.utcfromtimestamp(timestamp)) -def url_to_dependency_name(url): - return urlparse(url).netloc +# Validate GUID format +uuid_regex_pattern = re.compile('^[0-9a-f]{8}-' + '([0-9a-f]{4}-){3}' + '[0-9a-f]{12}$') + + +def validate_instrumentation_key(instrumentation_key): + """Validates the instrumentation key used for Azure Monitor. + + An instrumentation key cannot be null or empty. An instrumentation key + is valid for Azure Monitor only if it is a valid UUID. + + :param instrumentation_key: The instrumentation key to validate + """ + if not instrumentation_key: + raise ValueError("Instrumentation key cannot be none or empty.") + match = uuid_regex_pattern.match(instrumentation_key) + if not match: + raise ValueError("Invalid instrumentation key.") diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py index bf7c8163b..5ef12b3ba 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.3.dev0' +__version__ = '1.1.dev0' diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py index 4597c87fd..14e228d06 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py @@ -13,36 +13,94 @@ # limitations under the License. import logging +import random import threading import time import traceback -from opencensus.common.schedule import Queue -from opencensus.common.schedule import QueueExitEvent -from opencensus.common.schedule import QueueEvent -from opencensus.ext.azure.common import Options -from opencensus.ext.azure.common import utils -from opencensus.ext.azure.common.protocol import Data -from opencensus.ext.azure.common.protocol import Envelope -from opencensus.ext.azure.common.protocol import ExceptionData -from opencensus.ext.azure.common.protocol import Message +from opencensus.common.schedule import Queue, QueueEvent, QueueExitEvent +from opencensus.ext.azure.common import Options, utils +from opencensus.ext.azure.common.processor import ProcessorMixin +from opencensus.ext.azure.common.protocol import ( + Data, + Envelope, + Event, + ExceptionData, + Message, +) from opencensus.ext.azure.common.storage import LocalFileStorage -from opencensus.ext.azure.common.transport import TransportMixin +from opencensus.ext.azure.common.transport import ( + TransportMixin, + TransportStatusCode, +) +from opencensus.ext.azure.statsbeat import statsbeat +from opencensus.trace import execution_context logger = logging.getLogger(__name__) -__all__ = ['AzureLogHandler'] +__all__ = ['AzureEventHandler', 'AzureLogHandler'] class BaseLogHandler(logging.Handler): - def __init__(self): + + def __init__(self, **options): super(BaseLogHandler, self).__init__() - self._queue = Queue(capacity=8192) # TODO: make this configurable + self.options = Options(**options) + utils.validate_instrumentation_key(self.options.instrumentation_key) + if not 0 <= self.options.logging_sampling_rate <= 1: + raise ValueError('Sampling must be in the range: [0,1]') + self.export_interval = self.options.export_interval + self.max_batch_size = self.options.max_batch_size + self.storage = None + if self.options.enable_local_storage: + self.storage = LocalFileStorage( + path=self.options.storage_path, + max_size=self.options.storage_max_size, + maintenance_period=self.options.storage_maintenance_period, + retention_period=self.options.storage_retention_period, + source=self.__class__.__name__, + ) + self._telemetry_processors = [] + self.addFilter(SamplingFilter(self.options.logging_sampling_rate)) + self._queue = Queue(capacity=self.options.queue_capacity) self._worker = Worker(self._queue, self) self._worker.start() + # For redirects + self._consecutive_redirects = 0 # To prevent circular redirects + + def _export(self, batch, event=None): # pragma: NO COVER + try: + if batch: + envelopes = [self.log_record_to_envelope(x) for x in batch] + envelopes = self.apply_telemetry_processors(envelopes) + result = self._transmit(envelopes) + # Only store files if local storage enabled + if self.storage: + if result is TransportStatusCode.RETRY: + self.storage.put( + envelopes, + self.options.minimum_retry_interval, + ) + if result is TransportStatusCode.SUCCESS: + if len(batch) < self.options.max_batch_size: + self._transmit_from_storage() + if event: + if isinstance(event, QueueExitEvent): + # send files before exit + self._transmit_from_storage() + finally: + if event: + event.set() - def close(self): - self._worker.stop() + # Close is automatically called as part of logging shutdown + def close(self, timeout=None): + if not timeout and hasattr(self, "options"): + timeout = self.options.grace_period + if hasattr(self, "storage") and self.storage: + self.storage.close() + if hasattr(self, "_worker") and self._worker: + self._worker.stop(timeout) + super(BaseLogHandler, self).close() def createLock(self): self.lock = None @@ -50,17 +108,25 @@ def createLock(self): def emit(self, record): self._queue.put(record, block=False) - def _export(self, batch, event=None): - try: - return self.export(batch) - finally: - if event: - event.set() - - def export(self, batch): + def log_record_to_envelope(self, record): raise NotImplementedError # pragma: NO COVER + # Flush is automatically called as part of logging shutdown def flush(self, timeout=None): + if not hasattr(self, "_queue") or self._queue.is_empty(): + return + + # We must check the worker thread is alive, because otherwise flush + # is useless. Also, it would deadlock if no timeout is given, and the + # queue isn't empty. + # This is a very possible scenario during process termination, when + # atexit first calls handler.close() and then logging.shutdown(), + # that in turn calls handler.flush() without arguments. + if not self._worker.is_alive(): + logger.warning("Can't flush %s, worker thread is dead. " + "Any pending messages will be lost.", self) + return + self._queue.flush(timeout=timeout) @@ -76,6 +142,9 @@ def __init__(self, src, dst): ) def run(self): + # Indicate that this thread is an exporter thread. + # Used to suppress tracking of requests in this thread + execution_context.set_is_exporter(True) src = self._src dst = self._dst while True: @@ -105,62 +174,28 @@ def stop(self, timeout=None): # pragma: NO COVER return time.time() - start_time # time taken to stop -class AzureLogHandler(TransportMixin, BaseLogHandler): - """Handler for logging to Microsoft Azure Monitor. +class SamplingFilter(logging.Filter): - :param options: Options for the log handler. - """ + def __init__(self, probability=1.0): + super(SamplingFilter, self).__init__() + self.probability = probability - def __init__(self, **options): - self.options = Options(**options) - if not self.options.instrumentation_key: - raise ValueError('The instrumentation_key is not provided.') - self.export_interval = self.options.export_interval - self.max_batch_size = self.options.max_batch_size - self.storage = LocalFileStorage( - path=self.options.storage_path, - max_size=self.options.storage_max_size, - maintenance_period=self.options.storage_maintenance_period, - retention_period=self.options.storage_retention_period, - ) - super(AzureLogHandler, self).__init__() + def filter(self, record): + return random.random() < self.probability - def close(self): - self.storage.close() - super(AzureLogHandler, self).close() - def _export(self, batch, event=None): # pragma: NO COVER - try: - if batch: - envelopes = [self.log_record_to_envelope(x) for x in batch] - result = self._transmit(envelopes) - if result > 0: - self.storage.put(envelopes, result) - if event: - if isinstance(event, QueueExitEvent): - self._transmit_from_storage() # send files before exit - return - if len(batch) < self.options.max_batch_size: - self._transmit_from_storage() - finally: - if event: - event.set() +class AzureLogHandler(BaseLogHandler, TransportMixin, ProcessorMixin): + """Handler for logging to Microsoft Azure Monitor.""" + + def __init__(self, **options): + super(AzureLogHandler, self).__init__(**options) + # start statsbeat on exporter instantiation + if self._check_stats_collection(): + statsbeat.collect_statsbeat_metrics(self.options) def log_record_to_envelope(self, record): - envelope = Envelope( - iKey=self.options.instrumentation_key, - tags=dict(utils.azure_monitor_context), - time=utils.timestamp_to_iso_str(record.created), - ) - envelope.tags['ai.operation.id'] = getattr( - record, - 'traceId', - '00000000000000000000000000000000', - ) - envelope.tags['ai.operation.parentId'] = '|{}.{}.'.format( - envelope.tags['ai.operation.id'], - getattr(record, 'spanId', '0000000000000000'), - ) + envelope = create_envelope(self.options.instrumentation_key, record) + properties = { 'process': record.processName, 'module': record.module, @@ -168,28 +203,48 @@ def log_record_to_envelope(self, record): 'lineNumber': record.lineno, 'level': record.levelname, } + if (hasattr(record, 'custom_dimensions') and + isinstance(record.custom_dimensions, dict)): + properties.update(record.custom_dimensions) + if record.exc_info: exctype, _value, tb = record.exc_info callstack = [] level = 0 - for fileName, line, method, _text in traceback.extract_tb(tb): - callstack.append({ - 'level': level, - 'method': method, - 'fileName': fileName, - 'line': line, - }) - level += 1 - callstack.reverse() + has_full_stack = False + exc_type = "N/A" + message = self.format(record) + if tb is not None: + has_full_stack = True + for fileName, line, method, _text in traceback.extract_tb(tb): + callstack.append({ + 'level': level, + 'method': method, + 'fileName': fileName, + 'line': line, + }) + level += 1 + callstack.reverse() + elif record.message: + message = record.message + + if exctype is not None: + exc_type = exctype.__name__ + + if not exc_type: + exc_type = "Exception" + if not message: + message = "Exception" envelope.name = 'Microsoft.ApplicationInsights.Exception' + data = ExceptionData( exceptions=[{ 'id': 1, 'outerId': 0, - 'typeName': exctype.__name__, - 'message': self.format(record), - 'hasFullStack': True, + 'typeName': exc_type, + 'message': message, + 'hasFullStack': has_full_stack, 'parsedStack': callstack, }], severityLevel=max(0, record.levelno - 1) // 10, @@ -205,3 +260,55 @@ def log_record_to_envelope(self, record): ) envelope.data = Data(baseData=data, baseType='MessageData') return envelope + + +class AzureEventHandler(TransportMixin, ProcessorMixin, BaseLogHandler): + """Handler for sending custom events to Microsoft Azure Monitor.""" + + def __init__(self, **options): + super(AzureEventHandler, self).__init__(**options) + # start statsbeat on exporter instantiation + if self._check_stats_collection(): + statsbeat.collect_statsbeat_metrics(self.options) + + def log_record_to_envelope(self, record): + envelope = create_envelope(self.options.instrumentation_key, record) + + properties = {} + if (hasattr(record, 'custom_dimensions') and + isinstance(record.custom_dimensions, dict)): + properties.update(record.custom_dimensions) + + measurements = {} + if (hasattr(record, 'custom_measurements') and + isinstance(record.custom_measurements, dict)): + measurements.update(record.custom_measurements) + + envelope.name = 'Microsoft.ApplicationInsights.Event' + data = Event( + name=self.format(record), + properties=properties, + measurements=measurements, + ) + envelope.data = Data(baseData=data, baseType='EventData') + + return envelope + + +def create_envelope(instrumentation_key, record): + envelope = Envelope( + iKey=instrumentation_key, + tags=dict(utils.azure_monitor_context), + time=utils.timestamp_to_iso_str(record.created), + ) + envelope.tags['ai.operation.id'] = getattr( + record, + 'traceId', + '00000000000000000000000000000000', + ) + envelope.tags['ai.operation.parentId'] = '|{}.{}.'.format( + envelope.tags['ai.operation.id'], + getattr(record, 'spanId', '0000000000000000'), + ) + + return envelope diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py new file mode 100644 index 000000000..a92354138 --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py @@ -0,0 +1,190 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit + +from opencensus.common import utils as common_utils +from opencensus.ext.azure.common import Options, utils +from opencensus.ext.azure.common.processor import ProcessorMixin +from opencensus.ext.azure.common.protocol import ( + Data, + DataPoint, + Envelope, + MetricData, +) +from opencensus.ext.azure.common.storage import LocalFileStorage +from opencensus.ext.azure.common.transport import ( + TransportMixin, + TransportStatusCode, +) +from opencensus.ext.azure.metrics_exporter import standard_metrics +from opencensus.ext.azure.statsbeat.statsbeat_metrics import ( + _NETWORK_STATSBEAT_NAMES, +) +from opencensus.metrics import transport +from opencensus.metrics.export.metric_descriptor import MetricDescriptorType +from opencensus.stats import stats as stats_module + +__all__ = ['MetricsExporter', 'new_metrics_exporter'] + + +class MetricsExporter(TransportMixin, ProcessorMixin): + """Metrics exporter for Microsoft Azure Monitor.""" + + def __init__(self, is_stats=False, **options): + self.options = Options(**options) + self._is_stats = is_stats + utils.validate_instrumentation_key(self.options.instrumentation_key) + if self.options.max_batch_size <= 0: + raise ValueError('Max batch size must be at least 1.') + self.export_interval = self.options.export_interval + self.max_batch_size = self.options.max_batch_size + self._telemetry_processors = [] + self.storage = None + if self.options.enable_local_storage: + self.storage = LocalFileStorage( + path=self.options.storage_path, + max_size=self.options.storage_max_size, + maintenance_period=self.options.storage_maintenance_period, + retention_period=self.options.storage_retention_period, + source=self.__class__.__name__, + ) + self._atexit_handler = atexit.register(self.shutdown) + self.exporter_thread = None + # For redirects + self._consecutive_redirects = 0 # To prevent circular redirects + super(MetricsExporter, self).__init__() + + def export_metrics(self, metrics): + envelopes = [] + for metric in metrics: + envelopes.extend(self.metric_to_envelopes(metric)) + # Send data in batches of max_batch_size + batched_envelopes = list(common_utils.window( + envelopes, self.max_batch_size)) + for batch in batched_envelopes: + batch = self.apply_telemetry_processors(batch) + result = self._transmit(batch) + # If statsbeat exporter and received signal to shutdown + if self._is_stats and result is \ + TransportStatusCode.STATSBEAT_SHUTDOWN: + from opencensus.ext.azure.statsbeat import statsbeat + statsbeat.shutdown_statsbeat_metrics() + return + # Only store files if local storage enabled + if self.storage: + if result is TransportStatusCode.RETRY: + self.storage.put(batch, self.options.minimum_retry_interval) # noqa: E501 + if result is TransportStatusCode.SUCCESS: + # If there is still room to transmit envelopes, + # transmit from storage if available + if len(envelopes) < self.options.max_batch_size: + self._transmit_from_storage() + + def metric_to_envelopes(self, metric): + envelopes = [] + # No support for histogram aggregations + if (metric.descriptor.type != + MetricDescriptorType.CUMULATIVE_DISTRIBUTION): + md = metric.descriptor + # Each time series will be uniquely identified by its + # label values + for time_series in metric.time_series: + # time_series should only have one point which + # contains the aggregated value + # time_series point list is never empty + point = time_series.points[0] + # we ignore None and 0 values for network statsbeats + if self._is_stats_exporter(): + if md.name in _NETWORK_STATSBEAT_NAMES: + if not point.value.value: + continue + data_point = DataPoint( + ns=md.name, + name=md.name, + value=point.value.value + ) + # The timestamp is when the metric was recorded + timestamp = point.timestamp + # Get the properties using label keys from metric + # and label values of the time series + properties = self._create_properties( + time_series, + md.label_keys + ) + envelopes.append( + self._create_envelope( + data_point, + timestamp, + properties + ) + ) + return envelopes + + def _create_properties(self, time_series, label_keys): + properties = {} + # We construct a properties map from the label keys and values. We + # assume the ordering is already correct + for i in range(len(label_keys)): + if time_series.label_values[i].value is None: + value = "null" + else: + value = time_series.label_values[i].value + properties[label_keys[i].key] = value + return properties + + def _create_envelope(self, data_point, timestamp, properties): + envelope = Envelope( + iKey=self.options.instrumentation_key, + tags=dict(utils.azure_monitor_context), + time=timestamp.isoformat(), + ) + if self._is_stats: + envelope.name = "Statsbeat" + else: + envelope.name = "Microsoft.ApplicationInsights.Metric" + data = MetricData( + metrics=[data_point], + properties=properties + ) + envelope.data = Data(baseData=data, baseType="MetricData") + return envelope + + def shutdown(self): + if self.exporter_thread: + # flush if metrics exporter is not for stats + if not self._is_stats: + self.exporter_thread.close() + else: + self.exporter_thread.cancel() + # Shutsdown storage worker + if self.storage: + self.storage.close() + + +def new_metrics_exporter(**options): + exporter = MetricsExporter(**options) + producers = [stats_module.stats] + if exporter.options.enable_standard_metrics: + producers.append(standard_metrics.producer) + exporter.exporter_thread = transport.get_exporter_thread( + producers, + exporter, + interval=exporter.options.export_interval) + # start statsbeat on exporter instantiation + if exporter._check_stats_collection(): + # Import here to avoid circular dependencies + from opencensus.ext.azure.statsbeat import statsbeat + statsbeat.collect_statsbeat_metrics(exporter.options) + return exporter diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/__init__.py new file mode 100644 index 000000000..3ad5109ca --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/__init__.py @@ -0,0 +1,62 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from opencensus.ext.azure.metrics_exporter.standard_metrics.cpu import ( + ProcessorTimeMetric, +) +from opencensus.ext.azure.metrics_exporter.standard_metrics.http_requests import ( # noqa E501 + RequestsAvgExecutionMetric, + RequestsRateMetric, +) +from opencensus.ext.azure.metrics_exporter.standard_metrics.memory import ( + AvailableMemoryMetric, +) +from opencensus.ext.azure.metrics_exporter.standard_metrics.process import ( + ProcessCPUMetric, + ProcessMemoryMetric, +) +from opencensus.metrics.export.gauge import Registry +from opencensus.metrics.export.metric_producer import MetricProducer + +# List of standard metrics to track +STANDARD_METRICS = [AvailableMemoryMetric, + ProcessCPUMetric, + ProcessMemoryMetric, + ProcessorTimeMetric, + RequestsAvgExecutionMetric, + RequestsRateMetric] + + +def register_metrics(): + registry = Registry() + for standard_metric in STANDARD_METRICS: + metric = standard_metric() + registry.add_gauge(metric()) + return registry + + +class AzureStandardMetricsProducer(MetricProducer): + """Implementation of the producer of standard metrics. + + Includes Azure specific standard metrics, implemented + using gauges. + """ + def __init__(self): + self.registry = register_metrics() + + def get_metrics(self): + return self.registry.get_metrics() + + +producer = AzureStandardMetricsProducer() diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/cpu.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/cpu.py new file mode 100644 index 000000000..4f6226f0f --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/cpu.py @@ -0,0 +1,50 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import psutil + +from opencensus.metrics.export.gauge import DerivedDoubleGauge + + +class ProcessorTimeMetric(object): + NAME = "\\Processor(_Total)\\% Processor Time" + + @staticmethod + def get_value(): + cpu_times_percent = psutil.cpu_times_percent() + return 100 - cpu_times_percent.idle + + def __call__(self): + """ Returns a derived gauge for the processor time. + + Processor time is defined as a float representing the current system + wide CPU utilization minus idle CPU time as a percentage. Idle CPU + time is defined as the time spent doing nothing. Return values range + from 0.0 to 100.0 inclusive. + + :rtype: :class:`opencensus.metrics.export.gauge.DerivedDoubleGauge` + :return: The gauge representing the processor time metric + """ + gauge = DerivedDoubleGauge( + ProcessorTimeMetric.NAME, + 'Processor time as a percentage', + 'percentage', + []) + gauge.create_default_time_series(ProcessorTimeMetric.get_value) + # From the psutil docs: the first time this method is called with + # interval = None it will return a meaningless 0.0 value which you are + # supposed to ignore. Call cpu_percent() once so that the subsequent + # calls from the gauge will be meaningful. + psutil.cpu_times_percent() + return gauge diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/http_requests.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/http_requests.py new file mode 100644 index 000000000..629161dea --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/http_requests.py @@ -0,0 +1,176 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import threading +import time + +from opencensus.metrics.export.gauge import DerivedDoubleGauge + +if sys.version_info < (3,): + from BaseHTTPServer import HTTPServer +else: + from http.server import HTTPServer + +_requests_lock = threading.Lock() +requests_map = dict() +ORIGINAL_CONSTRUCTOR = HTTPServer.__init__ + + +def request_patch(func): + def wrapper(self=None): + start_time = time.time() + func(self) + end_time = time.time() + + update_request_state(start_time, end_time) + + return wrapper + + +def update_request_state(start_time, end_time): + # Update requests state information + # We don't want multiple threads updating this at once + with _requests_lock: + # Update Count + count = requests_map.get('count', 0) + requests_map['count'] = count + 1 + # Update duration + duration = requests_map.get('duration', 0) + requests_map['duration'] = duration + (end_time - start_time) + + +def server_patch(*args, **kwargs): + if len(args) >= 3: + handler = args[2] + if handler: + # Patch the handler methods if they exist + if "do_DELETE" in dir(handler): + handler.do_DELETE = request_patch(handler.do_DELETE) + if "do_GET" in dir(handler): + handler.do_GET = request_patch(handler.do_GET) + if "do_HEAD" in dir(handler): + handler.do_HEAD = request_patch(handler.do_HEAD) + if "do_OPTIONS" in dir(handler): + handler.do_OPTIONS = request_patch(handler.do_OPTIONS) + if "do_POST" in dir(handler): + handler.do_POST = request_patch(handler.do_POST) + if "do_PUT" in dir(handler): + handler.do_PUT = request_patch(handler.do_PUT) + result = ORIGINAL_CONSTRUCTOR(*args, **kwargs) + return result + + +def setup(): + # Patch the HTTPServer handler to track request information + HTTPServer.__init__ = server_patch + + +def get_average_execution_time(): + last_average_duration = requests_map.get('last_average_duration', 0) + interval_duration = requests_map.get('duration', 0) \ + - requests_map.get('last_duration', 0) + interval_count = requests_map.get('count', 0) \ + - requests_map.get('last_count', 0) + try: + result = interval_duration / interval_count + requests_map['last_average_duration'] = result + requests_map['last_duration'] = requests_map.get('duration', 0) + # Convert to milliseconds + return result * 1000.0 + except ZeroDivisionError: + # If interval_count is 0, exporter call made too close to previous + # Return the previous result if this is the case + return last_average_duration * 1000.0 + + +def get_requests_rate(): + current_time = time.time() + last_rate = requests_map.get('last_rate', 0) + last_time = requests_map.get('last_time') + + try: + # last_rate_time is None the first time this function is called + if last_time is not None: + interval_time = current_time - requests_map.get('last_time', 0) + interval_count = requests_map.get('count', 0) \ + - requests_map.get('last_count', 0) + result = interval_count / interval_time + else: + result = 0 + requests_map['last_time'] = current_time + requests_map['last_count'] = requests_map.get('count', 0) + requests_map['last_rate'] = result + return result + except ZeroDivisionError: + # If elapsed_seconds is 0, exporter call made too close to previous + # Return the previous result if this is the case + return last_rate + + +class RequestsAvgExecutionMetric(object): + NAME = "\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Request Execution Time" + + def __init__(self): + setup() + + @staticmethod + def get_value(): + return get_average_execution_time() + + def __call__(self): + """ Returns a derived gauge for incoming requests execution rate + + Calculated by getting the time it takes to make an incoming request + and dividing over the amount of incoming requests over an elapsed time. + + :rtype: :class:`opencensus.metrics.export.gauge.DerivedLongGauge` + :return: The gauge representing the incoming requests metric + """ + gauge = DerivedDoubleGauge( + RequestsAvgExecutionMetric.NAME, + 'Incoming Requests Average Execution Rate', + 'milliseconds', + []) + gauge.create_default_time_series(RequestsAvgExecutionMetric.get_value) + return gauge + + +class RequestsRateMetric(object): + NAME = "\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec" + + def __init__(self): + setup() + + @staticmethod + def get_value(): + return get_requests_rate() + + def __call__(self): + """ Returns a derived gauge for incoming requests per second + + Calculated by obtaining by getting the number of incoming requests + made to an HTTPServer within an elapsed time and dividing that value + over the elapsed time. + + :rtype: :class:`opencensus.metrics.export.gauge.DerivedLongGauge` + :return: The gauge representing the incoming requests metric + """ + gauge = DerivedDoubleGauge( + RequestsRateMetric.NAME, + 'Incoming Requests per second', + 'rps', + []) + gauge.create_default_time_series(RequestsRateMetric.get_value) + return gauge diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/memory.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/memory.py new file mode 100644 index 000000000..f24a7099d --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/memory.py @@ -0,0 +1,42 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import psutil + +from opencensus.metrics.export.gauge import DerivedLongGauge + + +class AvailableMemoryMetric(object): + NAME = "\\Memory\\Available Bytes" + + @staticmethod + def get_value(): + return psutil.virtual_memory().available + + def __call__(self): + """ Returns a derived gauge for available memory + + Available memory is defined as memory that can be given instantly to + processes without the system going into swap. + + :rtype: :class:`opencensus.metrics.export.gauge.DerivedLongGauge` + :return: The gauge representing the available memory metric + """ + gauge = DerivedLongGauge( + AvailableMemoryMetric.NAME, + 'Amount of available memory in bytes', + 'byte', + []) + gauge.create_default_time_series(AvailableMemoryMetric.get_value) + return gauge diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/process.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/process.py new file mode 100644 index 000000000..0222e7b74 --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/process.py @@ -0,0 +1,87 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import psutil + +from opencensus.metrics.export.gauge import ( + DerivedDoubleGauge, + DerivedLongGauge, +) + +logger = logging.getLogger(__name__) +PROCESS = psutil.Process() + + +class ProcessMemoryMetric(object): + NAME = "\\Process(??APP_WIN32_PROC??)\\Private Bytes" + + @staticmethod + def get_value(): + try: + return PROCESS.memory_info().rss + except Exception: + logger.exception('Error handling get process private bytes.') + + def __call__(self): + """ Returns a derived gauge for private bytes for the current process + + Private bytes for the current process is measured by the Resident Set + Size, which is the non-swapped physical memory a process has used. + + :rtype: :class:`opencensus.metrics.export.gauge.DerivedLongGauge` + :return: The gauge representing the private bytes metric + """ + gauge = DerivedLongGauge( + ProcessMemoryMetric.NAME, + 'Amount of memory process has used in bytes', + 'byte', + []) + gauge.create_default_time_series(ProcessMemoryMetric.get_value) + return gauge + + +class ProcessCPUMetric(object): + NAME = "\\Process(??APP_WIN32_PROC??)\\% Processor Time" + + @staticmethod + def get_value(): + try: + # In the case of a process running on multiple threads on different + # CPU cores, the returned value of cpu_percent() can be > 100.0. We + # normalize the cpu process using the number of logical CPUs + cpu_count = psutil.cpu_count(logical=True) + return PROCESS.cpu_percent() / cpu_count + except Exception: + logger.exception('Error handling get process cpu usage.') + + def __call__(self): + """ Returns a derived gauge for the CPU usage for the current process. + Return values range from 0.0 to 100.0 inclusive. + :rtype: :class:`opencensus.metrics.export.gauge.DerivedDoubleGauge` + :return: The gauge representing the process cpu usage metric + """ + gauge = DerivedDoubleGauge( + ProcessCPUMetric.NAME, + 'Process CPU usage as a percentage', + 'percentage', + []) + gauge.create_default_time_series(ProcessCPUMetric.get_value) + # From the psutil docs: the first time this method is called with + # interval = None it will return a meaningless 0.0 value which you are + # supposed to ignore. Call cpu_percent() with process once so that the + # subsequent calls from the gauge will be meaningful. + PROCESS.cpu_percent() + return gauge diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/state.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/state.py new file mode 100644 index 000000000..84ab6c71a --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/state.py @@ -0,0 +1,50 @@ +# Copyright 2020, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import threading + +_STATSBEAT_STATE = { + "INITIAL_FAILURE_COUNT": 0, + "INITIAL_SUCCESS": False, + "SHUTDOWN": False, +} +_STATSBEAT_STATE_LOCK = threading.Lock() + + +def is_statsbeat_enabled(): + disabled = os.environ.get("APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL") + return disabled is None or disabled.lower() != "true" + + +def increment_statsbeat_initial_failure_count(): + with _STATSBEAT_STATE_LOCK: + _STATSBEAT_STATE["INITIAL_FAILURE_COUNT"] += 1 + + +def get_statsbeat_initial_failure_count(): + return _STATSBEAT_STATE["INITIAL_FAILURE_COUNT"] + + +def set_statsbeat_initial_success(success): + with _STATSBEAT_STATE_LOCK: + _STATSBEAT_STATE["INITIAL_SUCCESS"] = success + + +def get_statsbeat_initial_success(): + return _STATSBEAT_STATE["INITIAL_SUCCESS"] + + +def get_statsbeat_shutdown(): + return _STATSBEAT_STATE["SHUTDOWN"] diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/statsbeat.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/statsbeat.py new file mode 100644 index 000000000..f0cfeed6a --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/statsbeat.py @@ -0,0 +1,100 @@ +# Copyright 2020, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import threading + +from opencensus.ext.azure.metrics_exporter import MetricsExporter +from opencensus.ext.azure.statsbeat.state import ( + _STATSBEAT_STATE, + _STATSBEAT_STATE_LOCK, +) +from opencensus.ext.azure.statsbeat.statsbeat_metrics import ( + _STATS_SHORT_EXPORT_INTERVAL, + _get_stats_connection_string, + _StatsbeatMetrics, +) +from opencensus.metrics import transport +from opencensus.metrics.export.metric_producer import MetricProducer +from opencensus.trace import execution_context + +_STATSBEAT_METRICS = None +_STATSBEAT_EXPORTER = None +_STATSBEAT_LOCK = threading.Lock() + + +def collect_statsbeat_metrics(options): + # pylint: disable=global-statement + global _STATSBEAT_METRICS + global _STATSBEAT_EXPORTER + # Only start statsbeat if did not exist before + if _STATSBEAT_METRICS is None and _STATSBEAT_EXPORTER is None: + with _STATSBEAT_LOCK: + # Only start statsbeat if did not exist before + exporter = MetricsExporter( + is_stats=True, + connection_string=_get_stats_connection_string(options.endpoint), # noqa: E501 + enable_local_storage=options.enable_local_storage, + enable_standard_metrics=False, + export_interval=_STATS_SHORT_EXPORT_INTERVAL, # 15m by default + ) + # The user's ikey is the one being tracked + producer = _AzureStatsbeatMetricsProducer(options) + _STATSBEAT_METRICS = producer + # Export some initial stats on program start + execution_context.set_is_exporter(True) + exporter.export_metrics(_STATSBEAT_METRICS.get_initial_metrics()) + execution_context.set_is_exporter(False) + exporter.exporter_thread = \ + transport.get_exporter_thread([_STATSBEAT_METRICS], + exporter, + exporter.options.export_interval) + _STATSBEAT_EXPORTER = exporter + with _STATSBEAT_STATE_LOCK: + _STATSBEAT_STATE["INITIAL_FAILURE_COUNT"] = 0 + _STATSBEAT_STATE["INITIAL_SUCCESS"] = 0 + _STATSBEAT_STATE["SHUTDOWN"] = False + + +def shutdown_statsbeat_metrics(): + # pylint: disable=global-statement + global _STATSBEAT_METRICS + global _STATSBEAT_EXPORTER + shutdown_success = False + if _STATSBEAT_METRICS is not None and _STATSBEAT_EXPORTER is not None and not _STATSBEAT_STATE["SHUTDOWN"]: # noqa: E501 + with _STATSBEAT_LOCK: + try: + _STATSBEAT_EXPORTER.shutdown() + _STATSBEAT_EXPORTER = None + _STATSBEAT_METRICS = None + shutdown_success = True + except: # pylint: disable=broad-except # noqa: E722 + pass + if shutdown_success: + with _STATSBEAT_STATE_LOCK: + _STATSBEAT_STATE["SHUTDOWN"] = True + + +class _AzureStatsbeatMetricsProducer(MetricProducer): + """Implementation of the producer of statsbeat metrics. + + Includes Azure attach rate, network and feature metrics, + implemented using gauges. + """ + def __init__(self, options): + self._statsbeat = _StatsbeatMetrics(options) + + def get_metrics(self): + return self._statsbeat.get_metrics() + + def get_initial_metrics(self): + return self._statsbeat.get_initial_metrics() diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/statsbeat_metrics.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/statsbeat_metrics.py new file mode 100644 index 000000000..712efecaf --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/statsbeat/statsbeat_metrics.py @@ -0,0 +1,480 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import json +import os +import platform +import re +import threading + +import requests + +from opencensus.ext.azure.common.transport import _requests_lock, _requests_map +from opencensus.ext.azure.common.version import __version__ as ext_version +from opencensus.metrics.export.gauge import ( + DerivedDoubleGauge, + DerivedLongGauge, + LongGauge, +) +from opencensus.metrics.label_key import LabelKey +from opencensus.metrics.label_value import LabelValue +from opencensus.trace.integrations import _Integrations, get_integrations + +_AIMS_URI = "http://169.254.169.254/metadata/instance/compute" +_AIMS_API_VERSION = "api-version=2017-12-01" +_AIMS_FORMAT = "format=json" + +_DEFAULT_NON_EU_STATS_CONNECTION_STRING = "InstrumentationKey=c4a29126-a7cb-47e5-b348-11414998b11e;IngestionEndpoint=https://westus-0.in.applicationinsights.azure.com/" # noqa: E501 +_DEFAULT_EU_STATS_CONNECTION_STRING = "InstrumentationKey=7dc56bab-3c0c-4e9f-9ebb-d1acadee8d0f;IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com/" # noqa: E501 +_DEFAULT_STATS_SHORT_EXPORT_INTERVAL = 900 # 15 minutes +_DEFAULT_STATS_LONG_EXPORT_INTERVAL = 86400 # 24 hours +_EU_ENDPOINTS = [ + "westeurope", + "northeurope", + "francecentral", + "francesouth", + "germanywestcentral", + "norwayeast", + "norwaywest", + "swedencentral", + "switzerlandnorth", + "switzerlandwest", + "uksouth", + "ukwest", +] + +_ATTACH_METRIC_NAME = "Attach" +_FEATURE_METRIC_NAME = "Feature" +_REQ_SUCCESS_NAME = "Request Success Count" +_REQ_FAILURE_NAME = "Request Failure Count" +_REQ_DURATION_NAME = "Request Duration" +_REQ_RETRY_NAME = "Retry Count" +_REQ_THROTTLE_NAME = "Throttle Count" +_REQ_EXCEPTION_NAME = "Exception Count" + +_NETWORK_STATSBEAT_NAMES = ( + _REQ_SUCCESS_NAME, + _REQ_FAILURE_NAME, + _REQ_DURATION_NAME, + _REQ_RETRY_NAME, + _REQ_THROTTLE_NAME, + _REQ_EXCEPTION_NAME, +) + +_ENDPOINT_TYPES = ["breeze"] +_RP_NAMES = ["appsvc", "functions", "vm", "unknown"] + +_HOST_PATTERN = re.compile('^https?://(?:www\\.)?([^/.]+)') + + +class _FEATURE_TYPES: + FEATURE = 0 + INSTRUMENTATION = 1 + + +class _StatsbeatFeature: + NONE = 0 + DISK_RETRY = 1 + AAD = 2 + + +def _get_stats_connection_string(endpoint): + cs_env = os.environ.get("APPLICATION_INSIGHTS_STATS_CONNECTION_STRING") + if cs_env: + return cs_env + else: + for ep in _EU_ENDPOINTS: + if ep in endpoint: + # Use statsbeat EU endpoint if user is in EU region + return _DEFAULT_EU_STATS_CONNECTION_STRING + return _DEFAULT_NON_EU_STATS_CONNECTION_STRING + + +def _get_stats_short_export_interval(): + ei_env = os.environ.get("APPLICATION_INSIGHTS_STATS_SHORT_EXPORT_INTERVAL") + if ei_env: + return int(ei_env) + else: + return _DEFAULT_STATS_SHORT_EXPORT_INTERVAL + + +def _get_stats_long_export_interval(): + ei_env = os.environ.get("APPLICATION_INSIGHTS_STATS_LONG_EXPORT_INTERVAL") + if ei_env: + return int(ei_env) + else: + return _DEFAULT_STATS_LONG_EXPORT_INTERVAL + + +_STATS_SHORT_EXPORT_INTERVAL = _get_stats_short_export_interval() +_STATS_LONG_EXPORT_INTERVAL = _get_stats_long_export_interval() +_STATS_LONG_INTERVAL_THRESHOLD = _STATS_LONG_EXPORT_INTERVAL / _STATS_SHORT_EXPORT_INTERVAL # noqa: E501 + + +def _get_common_properties(): + properties = [] + properties.append( + LabelKey("rp", 'name of the rp, e.g. appsvc, vm, function, aks, etc.')) + properties.append(LabelKey("attach", 'codeless or sdk')) + properties.append(LabelKey("cikey", 'customer ikey')) + properties.append(LabelKey("runtimeVersion", 'Python version')) + properties.append(LabelKey("os", 'os of application being instrumented')) + properties.append(LabelKey("language", 'python')) + properties.append(LabelKey("version", 'sdkVersion - version of the ext')) + return properties + + +def _get_attach_properties(): + properties = _get_common_properties() + properties.insert(1, LabelKey("rpId", 'unique id of rp')) + return properties + + +def _get_network_properties(value=None): + properties = _get_common_properties() + properties.append(LabelKey("endpoint", "ingestion endpoint type")) + properties.append(LabelKey("host", "destination of ingestion endpoint")) + if value is None: + properties.append(LabelKey("statusCode", "ingestion service response code")) # noqa: E501 + elif value == "Exception": + properties.append(LabelKey("exceptionType", "language specific exception type")) # noqa: E501 + return properties + + +def _get_feature_properties(): + properties = _get_common_properties() + properties.insert(4, LabelKey("feature", 'represents enabled features')) + properties.insert(4, LabelKey("type", 'type, either feature or instrumentation')) # noqa: E501 + return properties + + +def _get_success_count_value(): + with _requests_lock: + interval_count = _requests_map.get('success', 0) + _requests_map['success'] = 0 + return interval_count + + +def _get_failure_count_value(status_code): + interval_count = 0 + if status_code: + with _requests_lock: + if _requests_map.get('failure'): + interval_count = _requests_map.get('failure').get(status_code, 0) # noqa: E501 + _requests_map['failure'][status_code] = 0 + return interval_count + + +def _get_average_duration_value(): + with _requests_lock: + interval_duration = _requests_map.get('duration', 0) + interval_count = _requests_map.get('count', 0) + _requests_map['duration'] = 0 + _requests_map['count'] = 0 + if interval_duration > 0 and interval_count > 0: + result = interval_duration / interval_count + # Convert to milliseconds + return result * 1000.0 + return 0 + + +def _get_retry_count_value(status_code): + interval_count = 0 + if status_code: + with _requests_lock: + if _requests_map.get('retry'): + interval_count = _requests_map.get('retry').get(status_code, 0) + _requests_map['retry'][status_code] = 0 + return interval_count + + +def _get_throttle_count_value(status_code): + interval_count = 0 + if status_code: + with _requests_lock: + if _requests_map.get('throttle'): + interval_count = _requests_map.get('throttle').get(status_code, 0) # noqa: E501 + _requests_map['throttle'][status_code] = 0 + return interval_count + + +def _get_exception_count_value(exc_type): + interval_count = 0 + if exc_type: + with _requests_lock: + if _requests_map.get('exception'): + interval_count = _requests_map.get('exception').get(exc_type, 0) # noqa: E501 + _requests_map['exception'][exc_type] = 0 + return interval_count + + +def _shorten_host(host): + if not host: + host = "" + match = _HOST_PATTERN.match(host) + if match: + host = match.group(1) + return host + + +class _StatsbeatMetrics: + + def __init__(self, options): + self._options = options + self._instrumentation_key = options.instrumentation_key + self._feature = _StatsbeatFeature.NONE + if options.enable_local_storage: + self._feature |= _StatsbeatFeature.DISK_RETRY + if options.credential: + self._feature |= _StatsbeatFeature.AAD + self._stats_lock = threading.Lock() + self._vm_data = {} + self._vm_retry = True + self._rp = _RP_NAMES[3] + self._os_type = platform.system() + # Attach metrics - metrics related to rp (resource provider) + self._attach_metric = LongGauge( + _ATTACH_METRIC_NAME, + 'Statsbeat metric related to rp integrations', + 'count', + _get_attach_properties(), + ) + # Keep track of how many iterations until long export + self._long_threshold_count = 0 + # Network metrics - metrics related to request calls to Breeze + self._network_metrics = {} + # Map of gauge function -> metric + # Gauge function is the callback used to populate the metric value + self._network_metrics[_get_success_count_value] = DerivedLongGauge( + _REQ_SUCCESS_NAME, + 'Statsbeat metric tracking request success count', + 'count', + _get_network_properties(), + ) + self._network_metrics[_get_failure_count_value] = DerivedLongGauge( + _REQ_FAILURE_NAME, + 'Statsbeat metric tracking request failure count', + 'count', + _get_network_properties(), + ) + self._network_metrics[_get_average_duration_value] = DerivedDoubleGauge( # noqa: E501 + _REQ_DURATION_NAME, + 'Statsbeat metric tracking average request duration', + 'avg', + _get_network_properties(value="Duration"), + ) + self._network_metrics[_get_retry_count_value] = DerivedLongGauge( + _REQ_RETRY_NAME, + 'Statsbeat metric tracking request retry count', + 'count', + _get_network_properties(), + ) + self._network_metrics[_get_throttle_count_value] = DerivedLongGauge( + _REQ_THROTTLE_NAME, + 'Statsbeat metric tracking request throttle count', + 'count', + _get_network_properties(), + ) + self._network_metrics[_get_exception_count_value] = DerivedLongGauge( + _REQ_EXCEPTION_NAME, + 'Statsbeat metric tracking request exception count', + 'count', + _get_network_properties(value="Exception"), + ) + # feature/instrumentation metrics + # metrics related to what features and instrumentations are enabled + self._feature_metric = LongGauge( + _FEATURE_METRIC_NAME, + 'Statsbeat metric related to features enabled', # noqa: E501 + 'count', + _get_feature_properties(), + ) + # Instrumentation metric uses same name/properties as feature + self._instrumentation_metric = LongGauge( + _FEATURE_METRIC_NAME, + 'Statsbeat metric related to instrumentations enabled', # noqa: E501 + 'count', + _get_feature_properties(), + ) + + # Metrics that are sent on application start + def get_initial_metrics(self): + stats_metrics = [] + if self._attach_metric: + attach_metric = self._get_attach_metric() + if attach_metric: + stats_metrics.append(attach_metric) + if self._feature_metric: + feature_metric = self._get_feature_metric() + if feature_metric: + stats_metrics.append(feature_metric) + if self._instrumentation_metric: + instr_metric = self._get_instrumentation_metric() + if instr_metric: + stats_metrics.append(instr_metric) + return stats_metrics + + # Metrics sent every statsbeat interval + def get_metrics(self): + metrics = [] + try: + # Initial metrics use the long export interval + # Only export once long count hits threshold + with self._stats_lock: + self._long_threshold_count = self._long_threshold_count + 1 + if self._long_threshold_count >= _STATS_LONG_INTERVAL_THRESHOLD: # noqa: E501 + metrics.extend(self.get_initial_metrics()) + self._long_threshold_count = 0 + network_metrics = self._get_network_metrics() + metrics.extend(network_metrics) + except Exception: + pass + + return metrics + + def _get_network_metrics(self): + properties = self._get_common_properties() + properties.append(LabelValue(_ENDPOINT_TYPES[0])) # endpoint + host = _shorten_host(self._options.endpoint) + properties.append(LabelValue(host)) # host + metrics = [] + for fn, metric in self._network_metrics.items(): + if metric.descriptor.name == _REQ_SUCCESS_NAME: + properties.append(LabelValue(200)) + metric.create_time_series(properties, fn) + properties.pop() + elif metric.descriptor.name == _REQ_FAILURE_NAME: + for code in _requests_map.get('failure', {}).keys(): + properties.append(LabelValue(code)) + metric.create_time_series(properties, fn, status_code=code) + properties.pop() + elif metric.descriptor.name == _REQ_DURATION_NAME: + metric.create_time_series(properties, fn) + elif metric.descriptor.name == _REQ_RETRY_NAME: + for code in _requests_map.get('retry', {}).keys(): + properties.append(LabelValue(code)) + metric.create_time_series(properties, fn, status_code=code) + properties.pop() + elif metric.descriptor.name == _REQ_THROTTLE_NAME: + for code in _requests_map.get('throttle', {}).keys(): + properties.append(LabelValue(code)) + metric.create_time_series(properties, fn, status_code=code) + properties.pop() + elif metric.descriptor.name == _REQ_EXCEPTION_NAME: + for exc_type in _requests_map.get('exception', {}).keys(): + properties.append(LabelValue(exc_type)) + metric.create_time_series(properties, fn, exc_type=exc_type) # noqa: E501 + properties.pop() + + stats_metric = metric.get_metric(datetime.datetime.utcnow()) + # metric will be None if status_code or exc_type is invalid + # for success count, this will never be None + if stats_metric is not None: + # we handle not exporting of None and 0 values in the exporter + metrics.append(stats_metric) + return metrics + + def _get_feature_metric(self): + # Don't export if feature list is None + if self._feature is _StatsbeatFeature.NONE: + return None + properties = self._get_common_properties() + properties.insert(4, LabelValue(self._feature)) # feature long + properties.insert(4, LabelValue(_FEATURE_TYPES.FEATURE)) # type + self._feature_metric.get_or_create_time_series(properties) + return self._feature_metric.get_metric(datetime.datetime.utcnow()) + + def _get_instrumentation_metric(self): + integrations = get_integrations() + # Don't export if instrumentation list is None + if integrations is _Integrations.NONE: + return None + properties = self._get_common_properties() + properties.insert(4, LabelValue(get_integrations())) # instr long + properties.insert(4, LabelValue(_FEATURE_TYPES.INSTRUMENTATION)) # type # noqa: E501 + self._instrumentation_metric.get_or_create_time_series(properties) + return self._instrumentation_metric.get_metric(datetime.datetime.utcnow()) # noqa: E501 + + def _get_attach_metric(self): + properties = [] + rp = '' + rpId = '' + # rp, rpId + if os.environ.get("FUNCTIONS_WORKER_RUNTIME") is not None: + # Function apps + rp = _RP_NAMES[1] + rpId = os.environ.get("WEBSITE_HOSTNAME") + elif os.environ.get("WEBSITE_SITE_NAME") is not None: + # Web apps + rp = _RP_NAMES[0] + rpId = '{}/{}'.format( + os.environ.get("WEBSITE_SITE_NAME"), + os.environ.get("WEBSITE_HOME_STAMPNAME", '') + ) + elif self._vm_retry and self._get_azure_compute_metadata(): + # VM + rp = _RP_NAMES[2] + rpId = '{}/{}'.format( + self._vm_data.get("vmId", ''), + self._vm_data.get("subscriptionId", '')) + self._os_type = self._vm_data.get("osType", '') + else: + # Not in any rp or VM metadata failed + rp = _RP_NAMES[3] + rpId = _RP_NAMES[3] + + self._rp = rp + properties.extend(self._get_common_properties()) + properties.insert(1, LabelValue(rpId)) # rpid + self._attach_metric.get_or_create_time_series(properties) + return self._attach_metric.get_metric(datetime.datetime.utcnow()) + + def _get_common_properties(self): + properties = [] + properties.append(LabelValue(self._rp)) # rp + properties.append(LabelValue("sdk")) # attach type + properties.append(LabelValue(self._instrumentation_key)) # cikey + # runTimeVersion + properties.append(LabelValue(platform.python_version())) + properties.append(LabelValue(self._os_type or platform.system())) # os + properties.append(LabelValue("python")) # language + properties.append(LabelValue(ext_version)) # version + return properties + + def _get_azure_compute_metadata(self): + try: + request_url = "{0}?{1}&{2}".format( + _AIMS_URI, _AIMS_API_VERSION, _AIMS_FORMAT) + response = requests.get( + request_url, headers={"MetaData": "True"}, timeout=5.0) + except (requests.exceptions.ConnectionError, requests.Timeout): + # Not in VM + self._vm_retry = False + return False + except requests.exceptions.RequestException: + self._vm_retry = True # retry + return False + + try: + text = response.text + self._vm_data = json.loads(text) + except Exception: # pylint: disable=broad-except + # Error in reading response body, retry + self._vm_retry = True + return False + + # Vm data is perpetually updated + self._vm_retry = True + return True diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py index 52b9a54d3..627c1ed05 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py @@ -12,42 +12,75 @@ # See the License for the specific language governing permissions and # limitations under the License. +import atexit +import json import logging from opencensus.common.schedule import QueueExitEvent -from opencensus.ext.azure.common import Options -from opencensus.ext.azure.common import utils +from opencensus.ext.azure.common import Options, utils from opencensus.ext.azure.common.exporter import BaseExporter -from opencensus.ext.azure.common.protocol import Data -from opencensus.ext.azure.common.protocol import Envelope -from opencensus.ext.azure.common.protocol import RemoteDependency -from opencensus.ext.azure.common.protocol import Request +from opencensus.ext.azure.common.processor import ProcessorMixin +from opencensus.ext.azure.common.protocol import ( + Data, + Envelope, + ExceptionData, + RemoteDependency, + Request, +) from opencensus.ext.azure.common.storage import LocalFileStorage -from opencensus.ext.azure.common.transport import TransportMixin +from opencensus.ext.azure.common.transport import ( + TransportMixin, + TransportStatusCode, +) +from opencensus.ext.azure.statsbeat import statsbeat +from opencensus.trace import attributes_helper from opencensus.trace.span import SpanKind +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + logger = logging.getLogger(__name__) __all__ = ['AzureExporter'] +HTTP_METHOD = attributes_helper.COMMON_ATTRIBUTES['HTTP_METHOD'] +HTTP_PATH = attributes_helper.COMMON_ATTRIBUTES['HTTP_PATH'] +HTTP_ROUTE = attributes_helper.COMMON_ATTRIBUTES['HTTP_ROUTE'] +HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL'] +HTTP_STATUS_CODE = attributes_helper.COMMON_ATTRIBUTES['HTTP_STATUS_CODE'] +ERROR_MESSAGE = attributes_helper.COMMON_ATTRIBUTES['ERROR_MESSAGE'] +ERROR_NAME = attributes_helper.COMMON_ATTRIBUTES['ERROR_NAME'] +STACKTRACE = attributes_helper.COMMON_ATTRIBUTES['STACKTRACE'] + -class AzureExporter(TransportMixin, BaseExporter): +class AzureExporter(BaseExporter, TransportMixin, ProcessorMixin): """An exporter that sends traces to Microsoft Azure Monitor. :param options: Options for the exporter. """ def __init__(self, **options): - self.options = Options(**options) - if not self.options.instrumentation_key: - raise ValueError('The instrumentation_key is not provided.') - self.storage = LocalFileStorage( - path=self.options.storage_path, - max_size=self.options.storage_max_size, - maintenance_period=self.options.storage_maintenance_period, - retention_period=self.options.storage_retention_period, - ) super(AzureExporter, self).__init__(**options) + self.options = Options(**options) + utils.validate_instrumentation_key(self.options.instrumentation_key) + self.storage = None + if self.options.enable_local_storage: + self.storage = LocalFileStorage( + path=self.options.storage_path, + max_size=self.options.storage_max_size, + maintenance_period=self.options.storage_maintenance_period, + retention_period=self.options.storage_retention_period, + source=self.__class__.__name__, + ) + self._telemetry_processors = [] + atexit.register(self._stop, self.options.grace_period) + # start statsbeat on exporter instantiation + if self._check_stats_collection(): + statsbeat.collect_statsbeat_metrics(self.options) + # For redirects + self._consecutive_redirects = 0 # To prevent circular redirects def span_data_to_envelope(self, sd): envelope = Envelope( @@ -55,78 +88,149 @@ def span_data_to_envelope(self, sd): tags=dict(utils.azure_monitor_context), time=sd.start_time, ) + envelope.tags['ai.operation.id'] = sd.context.trace_id if sd.parent_span_id: - envelope.tags['ai.operation.parentId'] = '|{}.{}.'.format( - sd.context.trace_id, + envelope.tags['ai.operation.parentId'] = '{}'.format( sd.parent_span_id, ) if sd.span_kind == SpanKind.SERVER: + if ERROR_MESSAGE in sd.attributes: + message = sd.attributes.get(ERROR_MESSAGE) + if not message: + message = "Exception" + stack_trace = sd.attributes.get(STACKTRACE, []) + if not hasattr(stack_trace, '__iter__'): + stack_trace = [] + type_name = sd.attributes.get(ERROR_NAME, 'Exception') + exc_env = Envelope(**envelope) + exc_env.name = 'Microsoft.ApplicationInsights.Exception' + data = ExceptionData( + exceptions=[{ + 'id': 1, + 'outerId': 0, + 'typeName': type_name, + 'message': message, + 'hasFullStack': STACKTRACE in sd.attributes, + 'parsedStack': stack_trace + }], + ) + exc_env.data = Data(baseData=data, baseType='ExceptionData') + yield exc_env + envelope.name = 'Microsoft.ApplicationInsights.Request' data = Request( - id='|{}.{}.'.format(sd.context.trace_id, sd.span_id), + id='{}'.format(sd.span_id), duration=utils.timestamp_to_duration( sd.start_time, sd.end_time, ), - responseCode='0', # TODO - success=True, # TODO + responseCode=str(sd.status.code), + success=False, # Modify based off attributes or status + properties={}, ) envelope.data = Data(baseData=data, baseType='RequestData') - if 'http.method' in sd.attributes: - data.name = sd.attributes['http.method'] - if 'http.url' in sd.attributes: - data.name = data.name + ' ' + sd.attributes['http.url'] - data.url = sd.attributes['http.url'] - if 'http.status_code' in sd.attributes: - data.responseCode = str(sd.attributes['http.status_code']) + data.name = '' + if HTTP_METHOD in sd.attributes: + data.name = sd.attributes[HTTP_METHOD] + if HTTP_ROUTE in sd.attributes: + data.name = data.name + ' ' + sd.attributes[HTTP_ROUTE] + envelope.tags['ai.operation.name'] = data.name + data.properties['request.name'] = data.name + elif HTTP_PATH in sd.attributes: + data.properties['request.name'] = data.name + \ + ' ' + sd.attributes[HTTP_PATH] + if HTTP_URL in sd.attributes: + data.url = sd.attributes[HTTP_URL] + data.properties['request.url'] = sd.attributes[HTTP_URL] + if HTTP_STATUS_CODE in sd.attributes: + status_code = sd.attributes[HTTP_STATUS_CODE] + data.responseCode = str(status_code) + data.success = ( + status_code >= 200 and status_code <= 399 + ) + elif sd.status.code == 0: + data.success = True else: envelope.name = \ 'Microsoft.ApplicationInsights.RemoteDependency' data = RemoteDependency( name=sd.name, # TODO - id='|{}.{}.'.format(sd.context.trace_id, sd.span_id), - resultCode='0', # TODO + id='{}'.format(sd.span_id), + resultCode=str(sd.status.code), duration=utils.timestamp_to_duration( sd.start_time, sd.end_time, ), - success=True, # TODO + success=False, # Modify based off attributes or status + properties={}, ) envelope.data = Data( baseData=data, baseType='RemoteDependencyData', ) if sd.span_kind == SpanKind.CLIENT: - data.type = 'HTTP' # TODO - if 'http.url' in sd.attributes: - url = sd.attributes['http.url'] + data.type = sd.attributes.get('component') + if HTTP_URL in sd.attributes: + url = sd.attributes[HTTP_URL] # TODO: error handling, probably put scheme as well - data.name = utils.url_to_dependency_name(url) - if 'http.status_code' in sd.attributes: - data.resultCode = str(sd.attributes['http.status_code']) + data.data = url + parse_url = urlparse(url) + # target matches authority (host:port) + data.target = parse_url.netloc + if HTTP_METHOD in sd.attributes: + # name is METHOD/path + data.name = sd.attributes[HTTP_METHOD] \ + + ' ' + parse_url.path + if HTTP_STATUS_CODE in sd.attributes: + status_code = sd.attributes[HTTP_STATUS_CODE] + data.resultCode = str(status_code) + data.success = 200 <= status_code < 400 + elif sd.status.code == 0: + data.success = True else: data.type = 'INPROC' - # TODO: links, tracestate, tags, attrs - return envelope + data.success = True + if sd.links: + links = [] + for link in sd.links: + links.append( + {"operation_Id": link.trace_id, "id": link.span_id}) + data.properties["_MS.links"] = json.dumps(links) + # TODO: tracestate, tags + for key in sd.attributes: + # This removes redundant data from ApplicationInsights + if key.startswith('http.'): + continue + data.properties[key] = sd.attributes[key] + yield envelope def emit(self, batch, event=None): try: if batch: - envelopes = [self.span_data_to_envelope(sd) for sd in batch] + envelopes = [envelope + for sd in batch + for envelope in self.span_data_to_envelope(sd)] + envelopes = self.apply_telemetry_processors(envelopes) result = self._transmit(envelopes) - if result > 0: - self.storage.put(envelopes, result) + # Only store files if local storage enabled + if self.storage and result is TransportStatusCode.RETRY: + self.storage.put( + envelopes, + self.options.minimum_retry_interval + ) if event: - if isinstance(event, QueueExitEvent): + if self.storage and isinstance(event, QueueExitEvent): self._transmit_from_storage() # send files before exit event.set() return - if len(batch) < self.options.max_batch_size: + if self.storage and len(batch) < self.options.max_batch_size: self._transmit_from_storage() except Exception: logger.exception('Exception occurred while exporting the data.') def _stop(self, timeout=None): - self.storage.close() - return self._worker.stop(timeout) + if self.storage: + self.storage.close() + if self._worker: + self._worker.stop(timeout) diff --git a/contrib/opencensus-ext-azure/setup.py b/contrib/opencensus-ext-azure/setup.py index 2480969d5..422effebc 100644 --- a/contrib/opencensus-ext-azure/setup.py +++ b/contrib/opencensus-ext-azure/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from opencensus.ext.azure.common.version import __version__ setup( @@ -22,9 +22,9 @@ author='OpenCensus Authors', author_email='census-developers@googlegroups.com', classifiers=[ - 'Intended Audience :: Developers', - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', + 'Intended Audience :: End Users/Desktop', + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: End Users/Desktop', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', @@ -34,12 +34,17 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Azure Monitor Exporter', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'azure-core >= 1.12.0, < 2.0.0', + 'azure-identity >= 1.5.0, < 2.0.0', + 'opencensus >= 0.11.4, < 1.0.0', + 'psutil >= 5.6.3', 'requests >= 2.19.0', ], extras_require={}, diff --git a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py index 2707c9775..beba8ffdb 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py @@ -13,14 +13,16 @@ # limitations under the License. import logging -import mock import os import shutil import unittest +import mock + from opencensus.ext.azure import log_exporter +from opencensus.ext.azure.common.transport import TransportStatusCode -TEST_FOLDER = os.path.abspath('.test.logs') +TEST_FOLDER = os.path.abspath('.test.log.exporter') def setUpModule(): @@ -42,13 +44,31 @@ def __init__(self, max_batch_size, callback): self.export_interval = 1 self.max_batch_size = max_batch_size self.callback = callback - super(CustomLogHandler, self).__init__() + super(CustomLogHandler, self).__init__( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + ) def export(self, batch): return self.callback(batch) +class MockResponse(object): + def __init__(self, status_code, text, headers=None): + self.status_code = status_code + self.text = text + self.headers = headers + + class TestBaseLogHandler(unittest.TestCase): + + def setUp(self): + os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] = "true" + return super(TestBaseLogHandler, self).setUp() + + def tearDown(self): + del os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] + return super(TestBaseLogHandler, self).tearDown() + def test_basic(self): logger = logging.getLogger(self.id()) handler = CustomLogHandler(10, lambda batch: None) @@ -70,14 +90,54 @@ def test_export_exception(self): class TestAzureLogHandler(unittest.TestCase): + + def setUp(self): + os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] = "true" + return super(TestAzureLogHandler, self).setUp() + + def tearDown(self): + del os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] + return super(TestAzureLogHandler, self).tearDown() + def test_ctor(self): - from opencensus.ext.azure.common import Options - instrumentation_key = Options._default.instrumentation_key - Options._default.instrumentation_key = None - self.assertRaises(ValueError, lambda: log_exporter.AzureLogHandler()) - Options._default.instrumentation_key = instrumentation_key + self.assertRaises(ValueError, lambda: log_exporter.AzureLogHandler(connection_string="", instrumentation_key="")) # noqa: E501 + + def test_invalid_sampling_rate(self): + with self.assertRaises(ValueError): + log_exporter.AzureLogHandler( + enable_stats_metrics=False, + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=4.0, + ) + + def test_init_handler_with_proxies(self): + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + proxies='{"https":"https://test-proxy.com"}', + ) + + self.assertEqual( + handler.options.proxies, + '{"https":"https://test-proxy.com"}', + ) + + def test_init_handler_with_queue_capacity(self): + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + queue_capacity=500, + ) + + self.assertEqual( + handler.options.queue_capacity, + 500 + ) + + self.assertEqual( + handler._worker._src._queue.maxsize, + 500 + ) - @mock.patch('requests.post', return_value=mock.Mock()) + @mock.patch('requests.post', return_value=MockResponse(200, '')) def test_exception(self, requests_mock): logger = logging.getLogger(self.id()) handler = log_exporter.AzureLogHandler( @@ -94,7 +154,33 @@ def test_exception(self, requests_mock): post_body = requests_mock.call_args_list[0][1]['data'] self.assertTrue('ZeroDivisionError' in post_body) - @mock.patch('requests.post', return_value=mock.Mock()) + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_exception_with_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + try: + return 1 / 0 # generate a ZeroDivisionError + except Exception: + properties = { + 'custom_dimensions': + { + 'key_1': 'value_1', + 'key_2': 'value_2' + } + } + logger.exception('Captured an exception.', extra=properties) + handler.close() + self.assertEqual(len(requests_mock.call_args_list), 1) + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('ZeroDivisionError' in post_body) + self.assertTrue('key_1' in post_body) + self.assertTrue('key_2' in post_body) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) def test_export_empty(self, request_mock): handler = log_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', @@ -106,7 +192,7 @@ def test_export_empty(self, request_mock): @mock.patch('opencensus.ext.azure.log_exporter' '.AzureLogHandler.log_record_to_envelope') - def test_export_failure(self, log_record_to_envelope_mock): + def test_export_retry(self, log_record_to_envelope_mock): log_record_to_envelope_mock.return_value = ['bar'] handler = log_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', @@ -114,12 +200,28 @@ def test_export_failure(self, log_record_to_envelope_mock): ) with mock.patch('opencensus.ext.azure.log_exporter' '.AzureLogHandler._transmit') as transmit: - transmit.return_value = 10 + transmit.return_value = TransportStatusCode.RETRY handler._export(['foo']) self.assertEqual(len(os.listdir(handler.storage.path)), 1) self.assertIsNone(handler.storage.get()) handler.close() + @mock.patch('opencensus.ext.azure.log_exporter' + '.AzureLogHandler.log_record_to_envelope') + def test_export_success(self, log_record_to_envelope_mock): + log_record_to_envelope_mock.return_value = ['bar'] + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + with mock.patch('opencensus.ext.azure.log_exporter' + '.AzureLogHandler._transmit') as transmit: + transmit.return_value = TransportStatusCode.SUCCESS + handler._export(['foo']) + self.assertEqual(len(os.listdir(handler.storage.path)), 0) + self.assertIsNone(handler.storage.get()) + handler.close() + def test_log_record_to_envelope(self): handler = log_exporter.AzureLogHandler( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', @@ -133,3 +235,325 @@ def test_log_record_to_envelope(self): envelope.iKey, '12345678-1234-5678-abcd-12345678abcd') handler.close() + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_log_record_with_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + logger.warning('action', extra={ + 'custom_dimensions': + { + 'key_1': 'value_1', + 'key_2': 'value_2' + } + }) + handler.close() + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('action' in post_body) + self.assertTrue('key_1' in post_body) + self.assertTrue('key_2' in post_body) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_log_with_invalid_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + logger.warning('action_1_%s', None) + logger.warning('action_2_%s', 'arg', extra={ + 'custom_dimensions': 'not_a_dict' + }) + logger.warning('action_3_%s', 'arg', extra={ + 'notcustom_dimensions': {'key_1': 'value_1'} + }) + + handler.close() + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('action_1_' in post_body) + self.assertTrue('action_2_arg' in post_body) + self.assertTrue('action_3_arg' in post_body) + + self.assertFalse('not_a_dict' in post_body) + self.assertFalse('key_1' in post_body) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_log_record_sampled(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=1.0, + ) + logger.addHandler(handler) + logger.warning('Hello_World') + logger.warning('Hello_World2') + logger.warning('Hello_World3') + logger.warning('Hello_World4') + handler.close() + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('Hello_World' in post_body) + self.assertTrue('Hello_World2' in post_body) + self.assertTrue('Hello_World3' in post_body) + self.assertTrue('Hello_World4' in post_body) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_log_record_not_sampled(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=0.0, + ) + logger.addHandler(handler) + logger.warning('Hello_World') + logger.warning('Hello_World2') + logger.warning('Hello_World3') + logger.warning('Hello_World4') + handler.close() + self.assertTrue(handler._queue.is_empty()) + + +class TestAzureEventHandler(unittest.TestCase): + def setUp(self): + os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] = "true" + return super(TestAzureEventHandler, self).setUp() + + def tearDown(self): + del os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] + return super(TestAzureEventHandler, self).setUp() + + def test_ctor(self): + self.assertRaises(ValueError, lambda: log_exporter.AzureEventHandler(connection_string="", instrumentation_key="")) # noqa: E501 + + def test_invalid_sampling_rate(self): + with self.assertRaises(ValueError): + log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=4.0, + ) + + def test_init_handler_with_proxies(self): + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + proxies='{"https":"https://test-proxy.com"}', + ) + + self.assertEqual( + handler.options.proxies, + '{"https":"https://test-proxy.com"}', + ) + + def test_init_handler_with_queue_capacity(self): + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + queue_capacity=500, + ) + + self.assertEqual( + handler.options.queue_capacity, + 500 + ) + # pylint: disable=protected-access + self.assertEqual( + handler._worker._src._queue.maxsize, + 500 + ) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_exception(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + try: + return 1 / 0 # generate a ZeroDivisionError + except Exception: + logger.exception('Captured an exception.') + handler.close() + self.assertEqual(len(requests_mock.call_args_list), 1) + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('ZeroDivisionError' in post_body) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_exception_with_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + try: + return 1 / 0 # generate a ZeroDivisionError + except Exception: + properties = { + 'custom_dimensions': + { + 'key_1': 'value_1', + 'key_2': 'value_2' + }, + 'custom_measurements': + { + 'measure_1': 1, + 'measure_2': 2 + } + } + logger.exception('Captured an exception.', extra=properties) + handler.close() + self.assertEqual(len(requests_mock.call_args_list), 1) + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('ZeroDivisionError' in post_body) + self.assertTrue('key_1' in post_body) + self.assertTrue('key_2' in post_body) + self.assertTrue('measure_1' in post_body) + self.assertTrue('measure_2' in post_body) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_export_empty(self, request_mock): + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + handler._export([]) + self.assertEqual(len(os.listdir(handler.storage.path)), 0) + handler.close() + + @mock.patch('opencensus.ext.azure.log_exporter' + '.AzureEventHandler.log_record_to_envelope') + def test_export_retry(self, log_record_to_envelope_mock): + log_record_to_envelope_mock.return_value = ['bar'] + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + with mock.patch('opencensus.ext.azure.log_exporter' + '.AzureEventHandler._transmit') as transmit: + transmit.return_value = TransportStatusCode.RETRY + handler._export(['foo']) + self.assertEqual(len(os.listdir(handler.storage.path)), 1) + self.assertIsNone(handler.storage.get()) + handler.close() + + @mock.patch('opencensus.ext.azure.log_exporter' + '.AzureEventHandler.log_record_to_envelope') + def test_export_success(self, log_record_to_envelope_mock): + log_record_to_envelope_mock.return_value = ['bar'] + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + with mock.patch('opencensus.ext.azure.log_exporter' + '.AzureEventHandler._transmit') as transmit: + transmit.return_value = TransportStatusCode.SUCCESS + handler._export(['foo']) + self.assertEqual(len(os.listdir(handler.storage.path)), 0) + self.assertIsNone(handler.storage.get()) + handler.close() + + def test_log_record_to_envelope(self): + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + envelope = handler.log_record_to_envelope(mock.MagicMock( + exc_info=None, + levelno=10, + )) + self.assertEqual( + envelope.iKey, + '12345678-1234-5678-abcd-12345678abcd') + handler.close() + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_log_record_with_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + logger.warning('action', extra={ + 'custom_dimensions': + { + 'key_1': 'value_1', + 'key_2': 'value_2' + }, + 'custom_measurements': + { + 'measure_1': 1, + 'measure_2': 2 + } + }) + handler.close() + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('action' in post_body) + self.assertTrue('key_1' in post_body) + self.assertTrue('key_2' in post_body) + self.assertTrue('measure_1' in post_body) + self.assertTrue('measure_2' in post_body) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_log_with_invalid_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + logger.warning('action_1_%s', None) + logger.warning('action_2_%s', 'arg', extra={ + 'custom_dimensions': 'not_a_dict', + 'custom_measurements': 'also_not' + }) + logger.warning('action_3_%s', 'arg', extra={ + 'notcustom_dimensions': {'key_1': 'value_1'} + }) + + handler.close() + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('action_1_' in post_body) + self.assertTrue('action_2_arg' in post_body) + self.assertTrue('action_3_arg' in post_body) + + self.assertFalse('not_a_dict' in post_body) + self.assertFalse('also_not' in post_body) + self.assertFalse('key_1' in post_body) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_log_record_sampled(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=1.0, + ) + logger.addHandler(handler) + logger.warning('Hello_World') + logger.warning('Hello_World2') + logger.warning('Hello_World3') + logger.warning('Hello_World4') + handler.close() + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('Hello_World' in post_body) + self.assertTrue('Hello_World2' in post_body) + self.assertTrue('Hello_World3' in post_body) + self.assertTrue('Hello_World4' in post_body) + + @mock.patch('requests.post', return_value=MockResponse(200, '')) + def test_log_record_not_sampled(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=0.0, + ) + logger.addHandler(handler) + logger.warning('Hello_World') + logger.warning('Hello_World2') + logger.warning('Hello_World3') + logger.warning('Hello_World4') + handler.close() + self.assertFalse(requests_mock.called) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py new file mode 100644 index 000000000..a91c5eff4 --- /dev/null +++ b/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py @@ -0,0 +1,426 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +import unittest +from datetime import datetime + +import mock + +from opencensus.common import utils +from opencensus.ext.azure.common.protocol import DataPoint +from opencensus.ext.azure.common.transport import TransportStatusCode +from opencensus.ext.azure.metrics_exporter import ( + MetricsExporter, + new_metrics_exporter, + standard_metrics, +) +from opencensus.ext.azure.statsbeat.statsbeat_metrics import ( + _ATTACH_METRIC_NAME, + _REQ_SUCCESS_NAME, +) +from opencensus.metrics import label_key, label_value +from opencensus.metrics.export import ( + metric, + metric_descriptor, + point, + time_series, + value, +) +from opencensus.metrics.export.metric_descriptor import MetricDescriptorType + +TEST_FOLDER = os.path.abspath('.test.metrics.exporter') + + +def setUpModule(): + os.makedirs(TEST_FOLDER) + + +def tearDownModule(): + shutil.rmtree(TEST_FOLDER) + + +def create_metric(): + lv = label_value.LabelValue('val') + val = value.ValueLong(value=123) + dt = datetime(2019, 3, 20, 21, 34, 0, 537954) + pp = point.Point(value=val, timestamp=dt) + + ts = [ + time_series.TimeSeries(label_values=[lv], points=[pp], + start_timestamp=utils.to_iso_str(dt)) + ] + + desc = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + + mm = metric.Metric(descriptor=desc, time_series=ts) + return mm + + +def create_metric_ts(): + lv = label_value.LabelValue('val') + lv2 = label_value.LabelValue('val2') + val = value.ValueLong(value=123) + dt = datetime(2019, 3, 20, 21, 34, 0, 537954) + pp = point.Point(value=val, timestamp=dt) + + ts = [ + time_series.TimeSeries( + label_values=[lv], + points=[pp], + start_timestamp=utils.to_iso_str(dt) + ), + time_series.TimeSeries( + label_values=[lv2], + points=[pp], + start_timestamp=utils.to_iso_str(dt) + ), + ] + + desc = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + + mm = metric.Metric(descriptor=desc, time_series=ts) + return mm + + +def create_stats_metric(name, num): + lv = label_value.LabelValue('val') + val = value.ValueLong(value=num) + dt = datetime(2019, 3, 20, 21, 34, 0, 537954) + pp = point.Point(value=val, timestamp=dt) + + ts = [ + time_series.TimeSeries( + label_values=[lv], + points=[pp], + start_timestamp=utils.to_iso_str(dt) + ) + ] + + desc = metric_descriptor.MetricDescriptor( + name=name, + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + mm = metric.Metric(descriptor=desc, time_series=ts) + return mm + + +class TestAzureMetricsExporter(unittest.TestCase): + + def setUp(self): + os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] = "true" + return super(TestAzureMetricsExporter, self).setUp() + + def tearDown(self): + del os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] + return super(TestAzureMetricsExporter, self).tearDown() + + def test_constructor_missing_key(self): + self.assertRaises(ValueError, lambda: MetricsExporter(connection_string="", instrumentation_key="")) # noqa: E501 + + def test_constructor_invalid_batch_size(self): + self.assertRaises( + ValueError, + lambda: MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + max_batch_size=-1 + )) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_export_metrics(self, requests_mock): + metric = create_metric() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd') + requests_mock.return_value.text = '{"itemsReceived":1,'\ + '"itemsAccepted":1,'\ + '"errors":[]}' + requests_mock.return_value.status_code = 200 + exporter.export_metrics([metric]) + + self.assertEqual(len(requests_mock.call_args_list), 1) + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('metrics' in post_body) + self.assertTrue('properties' in post_body) + + def test_export_metrics_histogram(self): + metric = create_metric() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd') + metric.descriptor._type = MetricDescriptorType.CUMULATIVE_DISTRIBUTION + + self.assertIsNone(exporter.export_metrics([metric])) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_export_metrics_empty(self, requests_mock): + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + max_batch_size=1, + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + exporter.export_metrics([]) + + self.assertEqual(len(requests_mock.call_args_list), 0) + self.assertEqual(len(os.listdir(exporter.storage.path)), 0) + + def test_export_metrics_retry(self): + metric = create_metric() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + max_batch_size=1, + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + with mock.patch('opencensus.ext.azure.metrics_exporter' + '.MetricsExporter._transmit') as transmit: + transmit.return_value = TransportStatusCode.RETRY + exporter.export_metrics([metric]) + + self.assertEqual(len(os.listdir(exporter.storage.path)), 1) + self.assertIsNone(exporter.storage.get()) + + def test_export_metrics_success(self): + metric = create_metric() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + max_batch_size=1, + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + with mock.patch('opencensus.ext.azure.metrics_exporter' + '.MetricsExporter._transmit') as transmit: + transmit.return_value = TransportStatusCode.SUCCESS + exporter.export_metrics([metric]) + + self.assertEqual(len(os.listdir(exporter.storage.path)), 0) + self.assertIsNone(exporter.storage.get()) + + def test_metric_to_envelopes(self): + metric = create_metric() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + property_mock = mock.Mock() + envelope_mock = mock.Mock() + exporter._create_properties = property_mock + exporter._create_envelope = envelope_mock + exporter.metric_to_envelopes(metric) + property_mock.assert_called_once() + envelope_mock.assert_called_once() + + def test_metric_to_envelopes_multi_time_series(self): + metric = create_metric_ts() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + property_mock = mock.Mock() + envelope_mock = mock.Mock() + exporter._create_properties = property_mock + exporter._create_envelope = envelope_mock + exporter.metric_to_envelopes(metric) + property_mock.assert_any_call( + metric.time_series[0], + metric.descriptor.label_keys, + ) + property_mock.assert_any_call( + metric.time_series[1], + metric.descriptor.label_keys, + ) + envelope_mock.assert_called() + + def test_metric_to_envelopes_network_statsbeat(self): + metric = create_stats_metric(_REQ_SUCCESS_NAME, 10) + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + statsbeat_mock = mock.Mock() + statsbeat_mock.return_value = True + property_mock = mock.Mock() + envelope_mock = mock.Mock() + exporter._create_properties = property_mock + exporter._create_envelope = envelope_mock + exporter._is_stats_exporter = statsbeat_mock + exporter.metric_to_envelopes(metric) + property_mock.assert_called_once() + envelope_mock.assert_called_once() + + def test_metric_to_envelopes_network_statsbeat_zero(self): + metric = create_stats_metric(_REQ_SUCCESS_NAME, 0) + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + statsbeat_mock = mock.Mock() + statsbeat_mock.return_value = True + property_mock = mock.Mock() + envelope_mock = mock.Mock() + exporter._create_properties = property_mock + exporter._create_envelope = envelope_mock + exporter._is_stats_exporter = statsbeat_mock + exporter.metric_to_envelopes(metric) + property_mock.assert_not_called() + envelope_mock.assert_not_called() + + def test_metric_to_envelopes_not_network_statsbeat(self): + metric = create_stats_metric(_ATTACH_METRIC_NAME, 10) + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + statsbeat_mock = mock.Mock() + statsbeat_mock.return_value = True + property_mock = mock.Mock() + envelope_mock = mock.Mock() + exporter._create_properties = property_mock + exporter._create_envelope = envelope_mock + exporter._is_stats_exporter = statsbeat_mock + exporter.metric_to_envelopes(metric) + property_mock.assert_called_once() + envelope_mock.assert_called_once() + + def test_metric_to_envelopes_not_network_statsbeat_zero(self): + metric = create_stats_metric(_ATTACH_METRIC_NAME, 0) + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + statsbeat_mock = mock.Mock() + statsbeat_mock.return_value = True + property_mock = mock.Mock() + envelope_mock = mock.Mock() + exporter._create_properties = property_mock + exporter._create_envelope = envelope_mock + exporter._is_stats_exporter = statsbeat_mock + exporter.metric_to_envelopes(metric) + property_mock.assert_called_once() + envelope_mock.assert_called_once() + + def test_create_properties(self): + metric = create_metric() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + properties = exporter._create_properties(metric.time_series[0], + metric.descriptor.label_keys) + + self.assertEqual(len(properties), 1) + self.assertEqual(properties['key'], 'val') + + def test_create_properties_none(self): + metric = create_metric() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + metric.time_series[0].label_values[0]._value = None + properties = exporter._create_properties(metric.time_series[0], + metric.descriptor.label_keys) + + self.assertEqual(len(properties), 1) + self.assertEqual(properties['key'], 'null') + + def test_create_envelope(self): + metric = create_metric() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + value = metric.time_series[0].points[0].value.value + data_point = DataPoint(ns=metric.descriptor.name, + name=metric.descriptor.name, + value=value) + timestamp = datetime(2019, 3, 20, 21, 34, 0, 537954) + properties = {'url': 'website.com'} + envelope = exporter._create_envelope(data_point, timestamp, properties) + + self.assertTrue('iKey' in envelope) + self.assertEqual(envelope.iKey, '12345678-1234-5678-abcd-12345678abcd') + self.assertTrue('tags' in envelope) + self.assertTrue('time' in envelope) + self.assertEqual(envelope.time, timestamp.isoformat()) + self.assertTrue('name' in envelope) + self.assertEqual(envelope.name, 'Microsoft.ApplicationInsights.Metric') + self.assertTrue('data' in envelope) + self.assertTrue('baseData' in envelope.data) + self.assertTrue('baseType' in envelope.data) + self.assertTrue('metrics' in envelope.data.baseData) + self.assertTrue('properties' in envelope.data.baseData) + self.assertEqual(envelope.data.baseData.properties, properties) + + def test_shutdown(self): + mock_thread = mock.Mock() + mock_storage = mock.Mock() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + exporter.exporter_thread = mock_thread + exporter.storage = mock_storage + exporter.shutdown() + mock_thread.close.assert_called_once() + mock_storage.close.assert_called_once() + + def test_shutdown_statsbeat(self): + mock_thread = mock.Mock() + mock_storage = mock.Mock() + exporter = MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + exporter.exporter_thread = mock_thread + exporter._is_stats = True + exporter.storage = mock_storage + exporter.shutdown() + mock_thread.cancel.assert_called_once() + mock_storage.close.assert_called_once() + + @mock.patch('opencensus.ext.azure.metrics_exporter' + '.transport.get_exporter_thread') + def test_new_metrics_exporter(self, exporter_mock): + with mock.patch('opencensus.ext.azure.statsbeat' + '.statsbeat.collect_statsbeat_metrics') as hb: + hb.return_value = None + iKey = '12345678-1234-5678-abcd-12345678abcd' + exporter = new_metrics_exporter(instrumentation_key=iKey) + + self.assertEqual(exporter.options.instrumentation_key, iKey) + self.assertEqual(len(exporter_mock.call_args_list), 1) + self.assertEqual(len(exporter_mock.call_args[0][0]), 2) + producer_class = standard_metrics.AzureStandardMetricsProducer + self.assertFalse(isinstance(exporter_mock.call_args[0][0][0], + producer_class)) + self.assertTrue(isinstance(exporter_mock.call_args[0][0][1], + producer_class)) + + @mock.patch('opencensus.ext.azure.metrics_exporter' + '.transport.get_exporter_thread') + def test_new_metrics_exporter_no_standard_metrics(self, exporter_mock): + with mock.patch('opencensus.ext.azure.statsbeat' + '.statsbeat.collect_statsbeat_metrics') as hb: + hb.return_value = None + iKey = '12345678-1234-5678-abcd-12345678abcd' + exporter = new_metrics_exporter( + instrumentation_key=iKey, enable_standard_metrics=False) + + self.assertEqual(exporter.options.instrumentation_key, iKey) + self.assertEqual(len(exporter_mock.call_args_list), 1) + self.assertEqual(len(exporter_mock.call_args[0][0]), 1) + producer_class = standard_metrics.AzureStandardMetricsProducer + self.assertFalse(isinstance(exporter_mock.call_args[0][0][0], + producer_class)) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_standard_metrics.py b/contrib/opencensus-ext-azure/tests/test_azure_standard_metrics.py new file mode 100644 index 000000000..317c0e2c5 --- /dev/null +++ b/contrib/opencensus-ext-azure/tests/test_azure_standard_metrics.py @@ -0,0 +1,274 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import sys +import unittest + +import mock +import requests + +from opencensus.ext.azure.metrics_exporter import standard_metrics + +if sys.version_info < (3,): + from BaseHTTPServer import HTTPServer +else: + from http.server import HTTPServer + +ORIGINAL_FUNCTION = requests.Session.request +ORIGINAL_CONS = HTTPServer.__init__ + + +class TestStandardMetrics(unittest.TestCase): + def setUp(self): + standard_metrics.http_requests.requests_map.clear() + requests.Session.request = ORIGINAL_FUNCTION + standard_metrics.http_requests.ORIGINAL_CONSTRUCTOR = ORIGINAL_CONS + + @mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.register_metrics') + def test_producer_ctor(self, avail_mock): + standard_metrics.AzureStandardMetricsProducer() + + self.assertEqual(len(avail_mock.call_args_list), 1) + + def test_producer_get_metrics(self): + producer = standard_metrics.AzureStandardMetricsProducer() + metrics = producer.get_metrics() + + self.assertEqual(len(metrics), 6) + + def test_register_metrics(self): + registry = standard_metrics.register_metrics() + + self.assertEqual(len(registry.get_metrics()), 6) + + def test_get_available_memory_metric(self): + metric = standard_metrics.AvailableMemoryMetric() + gauge = metric() + + self.assertEqual(gauge.descriptor.name, '\\Memory\\Available Bytes') + + @mock.patch('psutil.virtual_memory') + def test_get_available_memory(self, psutil_mock): + memory = collections.namedtuple('memory', 'available') + vmem = memory(available=100) + psutil_mock.return_value = vmem + mem = standard_metrics.AvailableMemoryMetric.get_value() + + self.assertEqual(mem, 100) + + def test_get_process_private_bytes_metric(self): + metric = standard_metrics.ProcessMemoryMetric() + gauge = metric() + + # TODO: Refactor names to be platform generic + self.assertEqual(gauge.descriptor.name, + '\\Process(??APP_WIN32_PROC??)\\Private Bytes') + + def test_get_process_private_bytes(self): + with mock.patch('opencensus.ext.azure.metrics_exporter' + + '.standard_metrics.process.PROCESS') as process_mock: + memory = collections.namedtuple('memory', 'rss') + pmem = memory(rss=100) + process_mock.memory_info.return_value = pmem + mem = standard_metrics.ProcessMemoryMetric.get_value() + + self.assertEqual(mem, 100) + + @mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.process.logger') + def test_get_process_private_bytes_exception(self, logger_mock): + with mock.patch('opencensus.ext.azure.metrics_exporter' + + '.standard_metrics.process.PROCESS') as process_mock: + process_mock.memory_info.side_effect = Exception() + standard_metrics.ProcessMemoryMetric.get_value() + + logger_mock.exception.assert_called() + + def test_get_processor_time_metric(self): + metric = standard_metrics.ProcessorTimeMetric() + gauge = metric() + + self.assertEqual(gauge.descriptor.name, + '\\Processor(_Total)\\% Processor Time') + + def test_get_processor_time(self): + with mock.patch('psutil.cpu_times_percent') as processor_mock: + cpu = collections.namedtuple('cpu', 'idle') + cpu_times = cpu(idle=94.5) + processor_mock.return_value = cpu_times + processor_time = standard_metrics.ProcessorTimeMetric.get_value() + + self.assertEqual(processor_time, 5.5) + + def test_get_process_cpu_usage_metric(self): + metric = standard_metrics.ProcessCPUMetric() + gauge = metric() + + self.assertEqual(gauge.descriptor.name, + '\\Process(??APP_WIN32_PROC??)\\% Processor Time') + + @mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.process.psutil') + def test_get_process_cpu_usage(self, psutil_mock): + with mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.process.PROCESS') as process_mock: + process_mock.cpu_percent.return_value = 44.4 + psutil_mock.cpu_count.return_value = 2 + cpu_usage = standard_metrics.ProcessCPUMetric.get_value() + + self.assertEqual(cpu_usage, 22.2) + + @mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.process.logger') + def test_get_process_cpu_usage_exception(self, logger_mock): + with mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.process.psutil') as psutil_mock: + psutil_mock.cpu_count.return_value = None + standard_metrics.ProcessCPUMetric.get_value() + + logger_mock.exception.assert_called() + + def test_request_patch(self): + map = standard_metrics.http_requests.requests_map + func = mock.Mock() + new_func = standard_metrics.http_requests.request_patch(func) + new_func() + + self.assertEqual(map['count'], 1) + self.assertIsNotNone(map['duration']) + self.assertEqual(len(func.call_args_list), 1) + + def test_server_patch(self): + standard_metrics. \ + http_requests. \ + ORIGINAL_CONSTRUCTOR = lambda x, y, z: None + with mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.http_requests' + '.request_patch') as request_mock: + handler = mock.Mock() + handler.do_DELETE.return_value = None + handler.do_GET.return_value = None + handler.do_HEAD.return_value = None + handler.do_OPTIONS.return_value = None + handler.do_POST.return_value = None + handler.do_PUT.return_value = None + result = standard_metrics. \ + http_requests. \ + server_patch(None, None, handler) + handler.do_DELETE() + handler.do_GET() + handler.do_HEAD() + handler.do_OPTIONS() + handler.do_POST() + handler.do_PUT() + + self.assertEqual(result, None) + self.assertEqual(len(request_mock.call_args_list), 6) + + def test_server_patch_no_methods(self): + standard_metrics. \ + http_requests. \ + ORIGINAL_CONSTRUCTOR = lambda x, y, z: None + with mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.http_requests' + '.request_patch') as request_mock: + handler = mock.Mock() + result = standard_metrics. \ + http_requests. \ + server_patch(None, None, handler) + handler.do_DELETE() + handler.do_GET() + handler.do_HEAD() + handler.do_OPTIONS() + handler.do_POST() + handler.do_PUT() + + self.assertEqual(result, None) + self.assertEqual(len(request_mock.call_args_list), 0) + + def test_server_patch_no_args(self): + standard_metrics \ + .http_requests \ + .ORIGINAL_CONSTRUCTOR = lambda x, y: None + r = standard_metrics.http_requests.server_patch(None, None) + + self.assertEqual(r, None) + + def test_server_patch_no_handler(self): + standard_metrics \ + .http_requests \ + .ORIGINAL_CONSTRUCTOR = lambda x, y, z: None + r = standard_metrics.http_requests.server_patch(None, None, None) + + self.assertEqual(r, None) + + def test_get_requests_rate_metric(self): + metric = standard_metrics.RequestsRateMetric() + gauge = metric() + + name = '\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec' + self.assertEqual(gauge.descriptor.name, name) + + def test_get_requests_rate_first_time(self): + rate = standard_metrics.http_requests.get_requests_rate() + + self.assertEqual(rate, 0) + + @mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.http_requests.time') + def test_get_requests_rate(self, time_mock): + time_mock.time.return_value = 100 + standard_metrics.http_requests.requests_map['last_time'] = 98 + standard_metrics.http_requests.requests_map['count'] = 4 + rate = standard_metrics.http_requests.get_requests_rate() + + self.assertEqual(rate, 2) + + @mock.patch('opencensus.ext.azure.metrics_exporter' + '.standard_metrics.http_requests.time') + def test_get_requests_rate_error(self, time_mock): + time_mock.time.return_value = 100 + standard_metrics.http_requests.requests_map['last_rate'] = 5 + standard_metrics.http_requests.requests_map['last_time'] = 100 + result = standard_metrics.http_requests.get_requests_rate() + + self.assertEqual(result, 5) + + def test_get_requests_execution_metric(self): + metric = standard_metrics.RequestsAvgExecutionMetric() + gauge = metric() + + name = '\\ASP.NET Applications(??APP_W3SVC_PROC??)' \ + '\\Request Execution Time' + self.assertEqual(gauge.descriptor.name, name) + + def test_get_requests_execution(self): + map = standard_metrics.http_requests.requests_map + map['duration'] = 0.1 + map['count'] = 10 + map['last_count'] = 5 + result = standard_metrics.http_requests.get_average_execution_time() + + self.assertEqual(result, 20) + + def test_get_requests_execution_error(self): + map = standard_metrics.http_requests.requests_map + map['duration'] = 0.1 + map['count'] = 10 + map['last_count'] = 10 + result = standard_metrics.http_requests.get_average_execution_time() + + self.assertEqual(result, 0) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_statsbeat_metrics.py b/contrib/opencensus-ext-azure/tests/test_azure_statsbeat_metrics.py new file mode 100644 index 000000000..f5e4b6519 --- /dev/null +++ b/contrib/opencensus-ext-azure/tests/test_azure_statsbeat_metrics.py @@ -0,0 +1,848 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import platform +import unittest + +import mock +import requests + +from opencensus.ext.azure.common import Options +from opencensus.ext.azure.common.transport import _requests_map +from opencensus.ext.azure.common.version import __version__ as ext_version +from opencensus.ext.azure.metrics_exporter import MetricsExporter +from opencensus.ext.azure.statsbeat import statsbeat +from opencensus.ext.azure.statsbeat.statsbeat_metrics import ( + _DEFAULT_EU_STATS_CONNECTION_STRING, + _DEFAULT_NON_EU_STATS_CONNECTION_STRING, + _ENDPOINT_TYPES, + _FEATURE_TYPES, + _REQ_DURATION_NAME, + _REQ_EXCEPTION_NAME, + _REQ_FAILURE_NAME, + _REQ_RETRY_NAME, + _REQ_SUCCESS_NAME, + _REQ_THROTTLE_NAME, + _RP_NAMES, + _STATS_LONG_INTERVAL_THRESHOLD, + _get_attach_properties, + _get_average_duration_value, + _get_common_properties, + _get_exception_count_value, + _get_failure_count_value, + _get_feature_properties, + _get_network_properties, + _get_retry_count_value, + _get_stats_connection_string, + _get_success_count_value, + _get_throttle_count_value, + _shorten_host, + _StatsbeatMetrics, +) +from opencensus.metrics.export.gauge import ( + DerivedDoubleGauge, + DerivedLongGauge, + LongGauge, +) +from opencensus.trace import integrations + +_OPTIONS = Options( + instrumentation_key="ikey", + enable_local_storage=True, + endpoint="https://eastus-1.in.applicationinsights.azure.com/", + credential=None, +) + + +class MockResponse(object): + def __init__(self, status_code, text): + self.status_code = status_code + self.text = text + + +class MockCredential(object): + def get_token(): + pass + + +def throw(exc_type, *args, **kwargs): + def func(*_args, **_kwargs): + raise exc_type(*args, **kwargs) + return func + + +class TestStatsbeatMetrics(unittest.TestCase): + def setUp(self): + # pylint: disable=protected-access + statsbeat._STATSBEAT_METRICS = None + statsbeat._STATSBEAT_EXPORTER = None + _STATSBEAT_STATE = { # noqa: F841 + "INITIAL_FAILURE_COUNT": 0, + "INITIAL_SUCCESS": False, + "SHUTDOWN": False, + } + + def test_producer_ctor(self): + # pylint: disable=protected-access + producer = statsbeat._AzureStatsbeatMetricsProducer(_OPTIONS) + metrics = producer._statsbeat + self.assertTrue( + isinstance( + metrics, + _StatsbeatMetrics + ) + ) + self.assertEqual(metrics._instrumentation_key, "ikey") + + def test_producer_get_metrics(self): + # pylint: disable=protected-access + producer = statsbeat._AzureStatsbeatMetricsProducer(_OPTIONS) + mock_stats = mock.Mock() + producer._statsbeat = mock_stats + producer.get_metrics() + + mock_stats.get_metrics.assert_called_once() + + def test_producer_get_initial_metrics(self): + # pylint: disable=protected-access + producer = statsbeat._AzureStatsbeatMetricsProducer(_OPTIONS) + mock_stats = mock.Mock() + producer._statsbeat = mock_stats + producer.get_initial_metrics() + + mock_stats.get_initial_metrics.assert_called_once() + + @mock.patch.object(_StatsbeatMetrics, 'get_initial_metrics') + @mock.patch('opencensus.metrics.transport.get_exporter_thread') + def test_collect_statsbeat_metrics(self, thread_mock, stats_mock): + # pylint: disable=protected-access + self.assertIsNone(statsbeat._STATSBEAT_METRICS) + statsbeat.collect_statsbeat_metrics(_OPTIONS) + self.assertTrue( + isinstance( + statsbeat._STATSBEAT_METRICS, + statsbeat._AzureStatsbeatMetricsProducer + ) + ) + self.assertTrue( + isinstance( + statsbeat._STATSBEAT_EXPORTER, + MetricsExporter, + ) + ) + self.assertEqual( + statsbeat._STATSBEAT_METRICS._statsbeat._instrumentation_key, "ikey") # noqa: E501 + thread_mock.assert_called_once() + stats_mock.assert_called_once() + + @mock.patch.object(_StatsbeatMetrics, 'get_initial_metrics') + @mock.patch('opencensus.metrics.transport.get_exporter_thread') + def test_collect_statsbeat_metrics_exists(self, thread_mock, stats_mock): + # pylint: disable=protected-access + self.assertIsNone(statsbeat._STATSBEAT_METRICS) + producer = statsbeat._AzureStatsbeatMetricsProducer(_OPTIONS) + statsbeat._STATSBEAT_METRICS = producer + statsbeat.collect_statsbeat_metrics(None) + self.assertEqual(statsbeat._STATSBEAT_METRICS, producer) + thread_mock.assert_not_called() + stats_mock.assert_not_called() + + @mock.patch.object(_StatsbeatMetrics, 'get_initial_metrics') + @mock.patch('opencensus.metrics.transport.get_exporter_thread') + def test_collect_statsbeat_metrics_non_eu(self, thread_mock, stats_mock): + # pylint: disable=protected-access + cs = "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;IngestionEndpoint=https://westus-0.in.applicationinsights.azure.com/" # noqa: E501 + non_eu = Options( + connection_string=cs + ) + self.assertIsNone(statsbeat._STATSBEAT_METRICS) + with mock.patch.dict( + os.environ, { + "APPLICATION_INSIGHTS_STATS_CONNECTION_STRING": "", + }): + statsbeat.collect_statsbeat_metrics(non_eu) + self.assertTrue( + isinstance( + statsbeat._STATSBEAT_METRICS, + statsbeat._AzureStatsbeatMetricsProducer + ) + ) + self.assertTrue( + isinstance( + statsbeat._STATSBEAT_EXPORTER, + MetricsExporter, + ) + ) + self.assertEqual( + statsbeat._STATSBEAT_EXPORTER.options.instrumentation_key, # noqa: E501 + _DEFAULT_NON_EU_STATS_CONNECTION_STRING.split(";")[0].split("=")[1] # noqa: E501 + ) + self.assertEqual( + statsbeat._STATSBEAT_EXPORTER.options.endpoint, + _DEFAULT_NON_EU_STATS_CONNECTION_STRING.split(";")[1].split("=")[1] # noqa: E501 + ) + + @mock.patch.object(_StatsbeatMetrics, 'get_initial_metrics') + @mock.patch('opencensus.metrics.transport.get_exporter_thread') + def test_collect_statsbeat_metrics_eu(self, thread_mock, stats_mock): + # pylint: disable=protected-access + cs = "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;IngestionEndpoint=https://northeurope-0.in.applicationinsights.azure.com/" # noqa: E501 + eu = Options( + connection_string=cs + ) + with mock.patch.dict( + os.environ, { + "APPLICATION_INSIGHTS_STATS_CONNECTION_STRING": "", + }): + statsbeat.collect_statsbeat_metrics(eu) + self.assertTrue( + isinstance( + statsbeat._STATSBEAT_METRICS, + statsbeat._AzureStatsbeatMetricsProducer + ) + ) + self.assertTrue( + isinstance( + statsbeat._STATSBEAT_EXPORTER, + MetricsExporter, + ) + ) + self.assertEqual( + statsbeat._STATSBEAT_EXPORTER.options.instrumentation_key, # noqa: E501 + _DEFAULT_EU_STATS_CONNECTION_STRING.split(";")[0].split("=")[1] # noqa: E501 + ) + self.assertEqual( + statsbeat._STATSBEAT_EXPORTER.options.endpoint, + _DEFAULT_EU_STATS_CONNECTION_STRING.split(";")[1].split("=")[1] # noqa: E501 + ) + + def test_shutdown_statsbeat_metrics(self): + # pylint: disable=protected-access + producer_mock = mock.Mock() + exporter_mock = mock.Mock() + statsbeat._STATSBEAT_METRICS = producer_mock + statsbeat._STATSBEAT_EXPORTER = exporter_mock + statsbeat.shutdown_statsbeat_metrics() + exporter_mock.shutdown.assert_called_once() + self.assertIsNone(statsbeat._STATSBEAT_METRICS) + self.assertIsNone(statsbeat._STATSBEAT_EXPORTER) + + def test_shutdown_statsbeat_metrics_already_shutdown(self): + # pylint: disable=protected-access + producer_mock = mock.Mock() + exporter_mock = mock.Mock() + statsbeat._STATSBEAT_METRICS = producer_mock + statsbeat._STATSBEAT_EXPORTER = exporter_mock + statsbeat._STATSBEAT_STATE["SHUTDOWN"] = True + statsbeat.shutdown_statsbeat_metrics() + exporter_mock.shutdown.assert_not_called() + self.assertIsNotNone(statsbeat._STATSBEAT_METRICS) + self.assertIsNotNone(statsbeat._STATSBEAT_EXPORTER) + + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_feature_properties') # noqa: E501 + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_network_properties') # noqa: E501 + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_attach_properties') # noqa: E501 + def test_statsbeat_metric_init(self, attach_mock, network_mock, feature_mock): # noqa: E501 + # pylint: disable=protected-access + metric = _StatsbeatMetrics(_OPTIONS) + self.assertEqual(len(metric._vm_data), 0) + self.assertTrue(metric._vm_retry) + self.assertEqual(metric._instrumentation_key, "ikey") + self.assertEqual(metric._feature, 1) + self.assertTrue( + isinstance( + metric._attach_metric, + LongGauge, + ) + ) + self.assertTrue( + isinstance( + metric._network_metrics[_get_success_count_value], + DerivedLongGauge, + ) + ) + self.assertTrue( + isinstance( + metric._network_metrics[_get_failure_count_value], + DerivedLongGauge, + ) + ) + self.assertTrue( + isinstance( + metric._network_metrics[_get_average_duration_value], + DerivedDoubleGauge, + ) + ) + self.assertTrue( + isinstance( + metric._network_metrics[_get_retry_count_value], + DerivedLongGauge, + ) + ) + self.assertTrue( + isinstance( + metric._network_metrics[_get_throttle_count_value], + DerivedLongGauge, + ) + ) + self.assertTrue( + isinstance( + metric._network_metrics[_get_exception_count_value], + DerivedLongGauge, + ) + ) + self.assertTrue( + isinstance( + metric._feature_metric, + LongGauge, + ) + ) + self.assertTrue( + isinstance( + metric._instrumentation_metric, + LongGauge, + ) + ) + attach_mock.assert_called_once() + network_mock.assert_called() + self.assertEqual(feature_mock.call_count, 2) + attach_mock.assert_called_once() + network_mock.assert_called() + self.assertEqual(network_mock.call_count, 6) + + def test_get_attach_properties(self): + properties = _get_attach_properties() + self.assertEqual(properties[0].key, "rp") + self.assertEqual(properties[1].key, "rpId") + self.assertEqual(properties[2].key, "attach") + self.assertEqual(properties[3].key, "cikey") + self.assertEqual(properties[4].key, "runtimeVersion") + self.assertEqual(properties[5].key, "os") + self.assertEqual(properties[6].key, "language") + self.assertEqual(properties[7].key, "version") + + def test_get_feature_properties(self): + properties = _get_feature_properties() + self.assertEqual(properties[0].key, "rp") + self.assertEqual(properties[1].key, "attach") + self.assertEqual(properties[2].key, "cikey") + self.assertEqual(properties[3].key, "runtimeVersion") + self.assertEqual(properties[4].key, "type") + self.assertEqual(properties[5].key, "feature") + self.assertEqual(properties[6].key, "os") + self.assertEqual(properties[7].key, "language") + self.assertEqual(properties[8].key, "version") + + def test_get_network_properties(self): + properties = _get_network_properties() + self.assertEqual(properties[0].key, "rp") + self.assertEqual(properties[1].key, "attach") + self.assertEqual(properties[2].key, "cikey") + self.assertEqual(properties[3].key, "runtimeVersion") + self.assertEqual(properties[4].key, "os") + self.assertEqual(properties[5].key, "language") + self.assertEqual(properties[6].key, "version") + + def test_get_common_properties(self): + properties = _get_common_properties() + self.assertEqual(properties[0].key, "rp") + self.assertEqual(properties[1].key, "attach") + self.assertEqual(properties[2].key, "cikey") + self.assertEqual(properties[3].key, "runtimeVersion") + self.assertEqual(properties[4].key, "os") + self.assertEqual(properties[5].key, "language") + self.assertEqual(properties[6].key, "version") + + def test_get_success_count_value(self): + _requests_map.clear() + _requests_map['success'] = 10 + self.assertEqual(_get_success_count_value(), 10) + self.assertEqual(_requests_map['success'], 0) + _requests_map.clear() + + def test_get_failure_count_value(self): + _requests_map.clear() + _requests_map['failure'] = {} + _requests_map['failure'][400] = 10 + self.assertEqual(_get_failure_count_value(400), 10) + self.assertEqual(_requests_map['failure'][400], 0) + _requests_map.clear() + + def test_get_average_duration_value(self): + _requests_map.clear() + _requests_map['duration'] = 10 + _requests_map['count'] = 2 + self.assertEqual(_get_average_duration_value(), 5000.0) + self.assertEqual(_requests_map['duration'], 0) + self.assertEqual(_requests_map['count'], 0) + _requests_map.clear() + + def test_get_retry_count_value(self): + _requests_map.clear() + _requests_map['retry'] = {} + _requests_map['retry'][401] = 10 + self.assertEqual(_get_retry_count_value(401), 10) + self.assertEqual(_requests_map['retry'][401], 0) + _requests_map.clear() + + def test_get_throttle_count_value(self): + _requests_map.clear() + _requests_map['throttle'] = {} + _requests_map['throttle'][402] = 10 + self.assertEqual(_get_throttle_count_value(402), 10) + self.assertEqual(_requests_map['throttle'][402], 0) + _requests_map.clear() + + def test_get_exception_count_value(self): + _requests_map.clear() + _requests_map['exception'] = {} + _requests_map['exception']['Timeout'] = 10 + self.assertEqual(_get_exception_count_value('Timeout'), 10) + self.assertEqual(_requests_map['exception']['Timeout'], 0) + _requests_map.clear() + + def test_statsbeat_metric_get_initial_metrics(self): + # pylint: disable=protected-access + metric = _StatsbeatMetrics(_OPTIONS) + attach_metric_mock = mock.Mock() + attach_metric_mock.return_value = "attach" + feature_metric_mock = mock.Mock() + feature_metric_mock.return_value = "feature" + instr_metric_mock = mock.Mock() + instr_metric_mock.return_value = "instr" + metric._get_attach_metric = attach_metric_mock + metric._get_feature_metric = feature_metric_mock + metric._get_instrumentation_metric = instr_metric_mock + metrics = metric.get_initial_metrics() + attach_metric_mock.assert_called_once() + feature_metric_mock.assert_called_once() + instr_metric_mock.assert_called_once() + self.assertEqual(metrics, ["attach", "feature", "instr"]) + + def test_statsbeat_metric_get_metrics(self): + # pylint: disable=protected-access + metric = _StatsbeatMetrics(_OPTIONS) + metric._long_threshold_count = _STATS_LONG_INTERVAL_THRESHOLD + initial_metric_mock = mock.Mock() + network_metric_mock = mock.Mock() + initial_metric_mock.return_value = ["initial"] + network_metric_mock.return_value = ["network"] + metric.get_initial_metrics = initial_metric_mock + metric._get_network_metrics = network_metric_mock + metrics = metric.get_metrics() + initial_metric_mock.assert_called_once() + network_metric_mock.assert_called_once() + self.assertEqual(metrics, ["initial", "network"]) + self.assertEqual(metric._long_threshold_count, 0) + + def test_statsbeat_metric_get_metrics_short(self): + # pylint: disable=protected-access + metric = _StatsbeatMetrics(_OPTIONS) + metric._long_threshold_count = 1 + initial_metric_mock = mock.Mock() + network_metric_mock = mock.Mock() + initial_metric_mock.return_value = ["initial"] + network_metric_mock.return_value = ["network"] + metric.get_initial_metrics = initial_metric_mock + metric._get_network_metrics = network_metric_mock + metrics = metric.get_metrics() + initial_metric_mock.assert_not_called() + network_metric_mock.assert_called_once() + self.assertEqual(metrics, ["network"]) + self.assertEqual(metric._long_threshold_count, 2) + + def test_get_feature_metric(self): + stats = _StatsbeatMetrics(_OPTIONS) + metric = stats._get_feature_metric() + properties = metric._time_series[0]._label_values + self.assertEqual(len(properties), 9) + self.assertEqual(properties[0].value, _RP_NAMES[3]) + self.assertEqual(properties[1].value, "sdk") + self.assertEqual(properties[2].value, "ikey") + self.assertEqual(properties[3].value, platform.python_version()) + self.assertEqual(properties[4].value, _FEATURE_TYPES.FEATURE) + self.assertEqual(properties[5].value, 1) + self.assertEqual(properties[6].value, platform.system()) + self.assertEqual(properties[7].value, "python") + self.assertEqual( + properties[8].value, ext_version) # noqa: E501 + + def test_get_feature_metric_with_aad(self): + aad_options = Options( + instrumentation_key="ikey", + enable_local_storage=True, + endpoint="test-endpoint", + credential=MockCredential(), + ) + stats = _StatsbeatMetrics(aad_options) + metric = stats._get_feature_metric() + properties = metric._time_series[0]._label_values + self.assertEqual(len(properties), 9) + self.assertEqual(properties[0].value, _RP_NAMES[3]) + self.assertEqual(properties[1].value, "sdk") + self.assertEqual(properties[2].value, "ikey") + self.assertEqual(properties[3].value, platform.python_version()) + self.assertEqual(properties[4].value, _FEATURE_TYPES.FEATURE) + self.assertEqual(properties[5].value, 3) + self.assertEqual(properties[6].value, platform.system()) + self.assertEqual(properties[7].value, "python") + self.assertEqual( + properties[8].value, ext_version) # noqa: E501 + + def test_get_feature_metric_zero(self): + # pylint: disable=protected-access + options = Options( + instrumentation_key="ikey", + enable_local_storage=False, + credential=None, + ) + stats = _StatsbeatMetrics(options) + metric = stats._get_feature_metric() + self.assertIsNone(metric) + + def test_get_instrumentation_metric(self): + original_integrations = integrations._INTEGRATIONS_BIT_MASK + integrations._INTEGRATIONS_BIT_MASK = 1024 + stats = _StatsbeatMetrics(_OPTIONS) + metric = stats._get_instrumentation_metric() + properties = metric._time_series[0]._label_values + self.assertEqual(len(properties), 9) + self.assertEqual(properties[0].value, _RP_NAMES[3]) + self.assertEqual(properties[1].value, "sdk") + self.assertEqual(properties[2].value, "ikey") + self.assertEqual(properties[3].value, platform.python_version()) + self.assertEqual(properties[4].value, _FEATURE_TYPES.INSTRUMENTATION) + self.assertEqual(properties[5].value, 1024) + self.assertEqual(properties[6].value, platform.system()) + self.assertEqual(properties[7].value, "python") + self.assertEqual( + properties[8].value, ext_version) # noqa: E501 + integrations._INTEGRATIONS_BIT_MASK = original_integrations + + def test_get_instrumentation_metrics_zero(self): + # pylint: disable=protected-access + original_integrations = integrations._INTEGRATIONS_BIT_MASK + integrations._INTEGRATIONS_BIT_MASK = 0 + stats = _StatsbeatMetrics(_OPTIONS) + metric = stats._get_instrumentation_metric() + self.assertIsNone(metric) + integrations._INTEGRATIONS_BIT_MASK = original_integrations + + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_exception_count_value') # noqa: E501 + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_throttle_count_value') # noqa: E501 + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_retry_count_value') # noqa: E501 + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_average_duration_value') # noqa: E501 + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_failure_count_value') # noqa: E501 + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_success_count_value') # noqa: E501 + def test_get_network_metrics(self, mock1, mock2, mock3, mock4, mock5, mock6): # noqa: E501 + # pylint: disable=protected-access + _requests_map.clear() + _requests_map['exception'] = {} + _requests_map['throttle'] = {} + _requests_map['retry'] = {} + _requests_map['failure'] = {} + _requests_map['exception']['Timeout'] = 5 + _requests_map['exception']['RequestException'] = 5 + _requests_map['throttle'][402] = 5 + _requests_map['throttle'][439] = 5 + _requests_map['retry'][401] = 5 + _requests_map['retry'][403] = 5 + _requests_map['failure'][400] = 5 + _requests_map['failure'][404] = 5 + stats = _StatsbeatMetrics(_OPTIONS) + mock1.return_value = 5 + mock2.return_value = 5 + mock3.return_value = 5 + mock4.return_value = 5 + mock5.return_value = 5 + mock6.return_value = 5 + metrics = stats._get_network_metrics() + mock1.assert_called_once() + self.assertEqual(mock2.call_count, 2) + mock3.assert_called_once() + self.assertEqual(mock4.call_count, 2) + self.assertEqual(mock5.call_count, 2) + self.assertEqual(mock6.call_count, 2) + self.assertEqual(len(metrics), 6) + for metric in metrics: + for ts in metric._time_series: + properties = ts._label_values + if metric.descriptor.name == _REQ_DURATION_NAME: + self.assertEqual(len(properties), 9) + else: + self.assertEqual(len(properties), 10) + if metric.descriptor.name == _REQ_SUCCESS_NAME: + self.assertEqual(ts.points[0].value.value, 5) + self.assertEqual(properties[9].value, 200) + if metric.descriptor.name == _REQ_DURATION_NAME: + self.assertEqual(ts.points[0].value.value, 5) + if metric.descriptor.name == _REQ_FAILURE_NAME: + self.assertEqual(ts.points[0].value.value, 5) + self.assertTrue(properties[9].value in (400, 404)) + if metric.descriptor.name == _REQ_RETRY_NAME: + self.assertEqual(ts.points[0].value.value, 5) + self.assertTrue(properties[9].value in (401, 403)) + if metric.descriptor.name == _REQ_THROTTLE_NAME: + self.assertEqual(ts.points[0].value.value, 5) + self.assertTrue(properties[9].value in (402, 439)) + if metric.descriptor.name == _REQ_EXCEPTION_NAME: + self.assertEqual(ts.points[0].value.value, 5) + self.assertTrue(properties[9].value in ('Timeout', 'RequestException')) # noqa: E501 + self.assertEqual(properties[0].value, _RP_NAMES[3]) + self.assertEqual(properties[1].value, "sdk") + self.assertEqual(properties[2].value, "ikey") + self.assertEqual(properties[3].value, platform.python_version()) # noqa: E501 + self.assertEqual(properties[4].value, platform.system()) + self.assertEqual(properties[5].value, "python") + self.assertEqual(properties[6].value, ext_version) + self.assertEqual(properties[7].value, _ENDPOINT_TYPES[0]) + short_host = _shorten_host(_OPTIONS.endpoint) + self.assertEqual(properties[8].value, short_host) + _requests_map.clear() + + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_success_count_value') # noqa: E501 + @mock.patch( + 'opencensus.ext.azure.statsbeat.statsbeat_metrics._get_average_duration_value') # noqa: E501 + def test_get_network_metrics_zero(self, suc_mock, dur_mock): + # pylint: disable=protected-access + _requests_map.clear() + stats = _StatsbeatMetrics(_OPTIONS) + suc_mock.return_value = 0 + dur_mock.return_value = 0 + metrics = stats._get_network_metrics() + self.assertEqual(len(metrics), 2) + self.assertEqual(metrics[0]._time_series[0].points[0].value.value, 0) + self.assertEqual(metrics[1]._time_series[0].points[0].value.value, 0) + + @mock.patch.dict( + os.environ, + { + "WEBSITE_SITE_NAME": "site_name", + "WEBSITE_HOME_STAMPNAME": "stamp_name", + } + ) + def test_get_attach_metric_appsvc(self): + # pylint: disable=protected-access + stats = _StatsbeatMetrics(_OPTIONS) + metric = stats._get_attach_metric() + properties = metric._time_series[0]._label_values + self.assertEqual(len(properties), 8) + self.assertEqual(properties[0].value, _RP_NAMES[0]) + self.assertEqual(properties[1].value, "site_name/stamp_name") + self.assertEqual(properties[2].value, "sdk") + self.assertEqual(properties[3].value, "ikey") + self.assertEqual(properties[4].value, platform.python_version()) + self.assertEqual(properties[5].value, platform.system()) + self.assertEqual(properties[6].value, "python") + self.assertEqual( + properties[7].value, ext_version) # noqa: E501 + + @mock.patch.dict( + os.environ, + { + "FUNCTIONS_WORKER_RUNTIME": "runtime", + "WEBSITE_HOSTNAME": "host_name", + "WEBSITE_SITE_NAME": "site_name", + } + ) + def test_get_attach_metric_functions(self): + # pylint: disable=protected-access + stats = _StatsbeatMetrics(_OPTIONS) + metric = stats._get_attach_metric() + properties = metric._time_series[0]._label_values + self.assertEqual(len(properties), 8) + self.assertEqual(properties[0].value, _RP_NAMES[1]) + self.assertEqual(properties[1].value, "host_name") + self.assertEqual(properties[2].value, "sdk") + self.assertEqual(properties[3].value, "ikey") + self.assertEqual(properties[4].value, platform.python_version()) + self.assertEqual(properties[5].value, platform.system()) + self.assertEqual(properties[6].value, "python") + self.assertEqual( + properties[7].value, ext_version) # noqa: E501 + + def test_get_attach_metric_vm(self): + stats = _StatsbeatMetrics(_OPTIONS) + _vm_data = {} + _vm_data["vmId"] = "123" + _vm_data["subscriptionId"] = "sub123" + _vm_data["osType"] = "linux" + stats._vm_data = _vm_data + stats._vm_retry = True + metadata_mock = mock.Mock() + metadata_mock.return_value = True + stats._get_azure_compute_metadata = metadata_mock + metric = stats._get_attach_metric() + properties = metric._time_series[0]._label_values + self.assertEqual(len(properties), 8) + self.assertEqual(properties[0].value, _RP_NAMES[2]) + self.assertEqual(properties[1].value, "123/sub123") + self.assertEqual(properties[2].value, "sdk") + self.assertEqual(properties[3].value, "ikey") + self.assertEqual(properties[4].value, platform.python_version()) + self.assertEqual(properties[5].value, "linux") + self.assertEqual(properties[6].value, "python") + self.assertEqual( + properties[7].value, ext_version) # noqa: E501 + + def test_get_attach_metric_vm_no_os(self): + stats = _StatsbeatMetrics(_OPTIONS) + _vm_data = {} + _vm_data["vmId"] = "123" + _vm_data["subscriptionId"] = "sub123" + _vm_data["osType"] = None + stats._vm_data = _vm_data + stats._vm_retry = True + metadata_mock = mock.Mock() + metadata_mock.return_value = True + stats._get_azure_compute_metadata = metadata_mock + metric = stats._get_attach_metric() + properties = metric._time_series[0]._label_values + self.assertEqual(len(properties), 8) + self.assertEqual(properties[5].value, platform.system()) + + def test_get_attach_metric_unknown(self): + stats = _StatsbeatMetrics(_OPTIONS) + stats._vm_retry = False + metric = stats._get_attach_metric() + properties = metric._time_series[0]._label_values + self.assertEqual(len(properties), 8) + self.assertEqual(properties[0].value, _RP_NAMES[3]) + self.assertEqual(properties[1].value, _RP_NAMES[3]) + self.assertEqual(properties[2].value, "sdk") + self.assertEqual(properties[3].value, "ikey") + self.assertEqual(properties[4].value, platform.python_version()) + self.assertEqual(properties[5].value, platform.system()) + self.assertEqual(properties[6].value, "python") + self.assertEqual( + properties[7].value, ext_version) # noqa: E501 + + def test_get_azure_compute_metadata(self): + with mock.patch('requests.get') as get: + get.return_value = MockResponse( + 200, + json.dumps( + { + 'vmId': 5, + 'subscriptionId': 3, + 'osType': 'Linux' + } + ) + ) + stats = _StatsbeatMetrics(_OPTIONS) + vm_result = stats._get_azure_compute_metadata() + self.assertTrue(vm_result) + self.assertEqual(stats._vm_data["vmId"], 5) + self.assertEqual(stats._vm_data["subscriptionId"], 3) + self.assertEqual(stats._vm_data["osType"], "Linux") + self.assertTrue(stats._vm_retry) + + def test_get_azure_compute_metadata_not_vm(self): + with mock.patch( + 'requests.get', + throw(requests.exceptions.ConnectionError) + ): + stats = _StatsbeatMetrics(_OPTIONS) + vm_result = stats._get_azure_compute_metadata() + self.assertFalse(vm_result) + self.assertEqual(len(stats._vm_data), 0) + self.assertFalse(stats._vm_retry) + + def test_get_azure_compute_metadata_not_vm_timeout(self): + with mock.patch( + 'requests.get', + throw(requests.Timeout) + ): + stats = _StatsbeatMetrics(_OPTIONS) + vm_result = stats._get_azure_compute_metadata() + self.assertFalse(vm_result) + self.assertEqual(len(stats._vm_data), 0) + self.assertFalse(stats._vm_retry) + + def test_get_azure_compute_metadata_vm_retry(self): + with mock.patch( + 'requests.get', + throw(requests.exceptions.RequestException) + ): + stats = _StatsbeatMetrics(_OPTIONS) + vm_result = stats._get_azure_compute_metadata() + self.assertFalse(vm_result) + self.assertEqual(len(stats._vm_data), 0) + self.assertTrue(stats._vm_retry) + + def test_shorten_host(self): + url = "https://fakehost-1.example.com/" + self.assertEqual(_shorten_host(url), "fakehost-1") + url = "https://fakehost-2.example.com/" + self.assertEqual(_shorten_host(url), "fakehost-2") + url = "http://www.fakehost-3.example.com/" + self.assertEqual(_shorten_host(url), "fakehost-3") + url = "http://www.fakehost.com/v2/track" + self.assertEqual(_shorten_host(url), "fakehost") + url = "https://www.fakehost0-4.com/" + self.assertEqual(_shorten_host(url), "fakehost0-4") + url = "https://www.fakehost-5.com" + self.assertEqual(_shorten_host(url), "fakehost-5") + url = "https://fakehost.com" + self.assertEqual(_shorten_host(url), "fakehost") + url = "http://fakehost-5/" + self.assertEqual(_shorten_host(url), "fakehost-5") + + def test_get_stats_connection_string_env(self): + cs = "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;IngestionEndpoint=https://westus-0.in.applicationinsights.azure.com/" # noqa: E501 + with mock.patch.dict( + os.environ, { + "APPLICATION_INSIGHTS_STATS_CONNECTION_STRING": cs + } + ): + stats_cs = _get_stats_connection_string(_OPTIONS.endpoint) + self.assertEqual(stats_cs, cs) + + def test_get_stats_connection_string_non_eu(self): + with mock.patch.dict( + os.environ, { + "APPLICATION_INSIGHTS_STATS_CONNECTION_STRING": "" + } + ): + cs = "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;IngestionEndpoint=https://westus-0.in.applicationinsights.azure.com/" # noqa: E501 + non_eu = Options( + connection_string=cs, + ) + stats_cs = _get_stats_connection_string(non_eu.endpoint) + self.assertEqual(stats_cs, _DEFAULT_NON_EU_STATS_CONNECTION_STRING) + + def test_get_stats_connection_string_eu(self): + with mock.patch.dict( + os.environ, { + "APPLICATION_INSIGHTS_STATS_CONNECTION_STRING": "" + } + ): + cs = "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;IngestionEndpoint=https://northeurope-0.in.applicationinsights.azure.com/" # noqa: E501 + eu = Options( + connection_string=cs, + ) + stats_cs = _get_stats_connection_string(eu.endpoint) + self.assertEqual(stats_cs, _DEFAULT_EU_STATS_CONNECTION_STRING) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py index 207cd092c..c8bbd9185 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py @@ -13,14 +13,17 @@ # limitations under the License. import json -import mock import os import shutil import unittest +import mock + from opencensus.ext.azure import trace_exporter +from opencensus.ext.azure.common.transport import TransportStatusCode +from opencensus.trace.link import Link -TEST_FOLDER = os.path.abspath('.test.exporter') +TEST_FOLDER = os.path.abspath('.test.trace.exporter') def setUpModule(): @@ -38,12 +41,44 @@ def func(*_args, **_kwargs): class TestAzureExporter(unittest.TestCase): + + def setUp(self): + os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] = "true" + return super(TestAzureExporter, self).setUp() + + def tearDown(self): + del os.environ["APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL"] + return super(TestAzureExporter, self).tearDown() + def test_ctor(self): - from opencensus.ext.azure.common import Options - instrumentation_key = Options._default.instrumentation_key - Options._default.instrumentation_key = None - self.assertRaises(ValueError, lambda: trace_exporter.AzureExporter()) - Options._default.instrumentation_key = instrumentation_key + self.assertRaises(ValueError, lambda: trace_exporter.AzureExporter(connection_string="", instrumentation_key="")) # noqa: E501 + + def test_init_exporter_with_proxies(self): + exporter = trace_exporter.AzureExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + proxies='{"https":"https://test-proxy.com"}', + ) + + self.assertEqual( + exporter.options.proxies, + '{"https":"https://test-proxy.com"}', + ) + + def test_init_exporter_with_queue_capacity(self): + exporter = trace_exporter.AzureExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + queue_capacity=500, + ) + + self.assertEqual( + exporter.options.queue_capacity, + 500 + ) + # pylint: disable=protected-access + self.assertEqual( + exporter._worker.src._queue.maxsize, + 500 + ) @mock.patch('requests.post', return_value=mock.Mock()) def test_emit_empty(self, request_mock): @@ -66,14 +101,14 @@ def test_emit_exception(self, mock_logger): exporter._stop() @mock.patch('opencensus.ext.azure.trace_exporter.AzureExporter.span_data_to_envelope') # noqa: E501 - def test_emit_failure(self, span_data_to_envelope_mock): + def test_emit_retry(self, span_data_to_envelope_mock): span_data_to_envelope_mock.return_value = ['bar'] exporter = trace_exporter.AzureExporter( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', storage_path=os.path.join(TEST_FOLDER, self.id()), ) with mock.patch('opencensus.ext.azure.trace_exporter.AzureExporter._transmit') as transmit: # noqa: E501 - transmit.return_value = 10 + transmit.return_value = TransportStatusCode.RETRY exporter.emit(['foo']) self.assertEqual(len(os.listdir(exporter.storage.path)), 1) self.assertIsNone(exporter.storage.get()) @@ -88,7 +123,7 @@ def test_emit_success(self, span_data_to_envelope_mock): storage_path=os.path.join(TEST_FOLDER, self.id()), ) with mock.patch('opencensus.ext.azure.trace_exporter.AzureExporter._transmit') as transmit: # noqa: E501 - transmit.return_value = 0 + transmit.return_value = TransportStatusCode.SUCCESS exporter.emit([]) exporter.emit(['foo']) self.assertEqual(len(os.listdir(exporter.storage.path)), 0) @@ -100,6 +135,7 @@ def test_span_data_to_envelope(self): from opencensus.trace.span import SpanKind from opencensus.trace.span_context import SpanContext from opencensus.trace.span_data import SpanData + from opencensus.trace.status import Status from opencensus.trace.trace_options import TraceOptions from opencensus.trace.tracestate import Tracestate @@ -109,7 +145,7 @@ def test_span_data_to_envelope(self): ) # SpanKind.CLIENT HTTP - envelope = exporter.span_data_to_envelope(SpanData( + envelope = next(exporter.span_data_to_envelope(SpanData( name='test', context=SpanContext( trace_id='6e0c63257de34c90bf9efcd03927272e', @@ -121,6 +157,7 @@ def test_span_data_to_envelope(self): span_id='6e0c63257de34c92', parent_span_id='6e0c63257de34c93', attributes={ + 'component': 'HTTP', 'http.method': 'GET', 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', 'http.status_code': 200, @@ -128,14 +165,16 @@ def test_span_data_to_envelope(self): start_time='2010-10-24T07:28:38.123456Z', end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, - links=None, - status=None, + links=[ + Link('6e0c63257de34c90bf9efcd03927272e', '6e0c63257de34c91') + ], + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, child_span_count=None, span_kind=SpanKind.CLIENT, - )) + ))) self.assertEqual( envelope.iKey, '12345678-1234-5678-abcd-12345678abcd') @@ -144,7 +183,7 @@ def test_span_data_to_envelope(self): 'Microsoft.ApplicationInsights.RemoteDependency') self.assertEqual( envelope.tags['ai.operation.parentId'], - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c93.') + '6e0c63257de34c93') self.assertEqual( envelope.tags['ai.operation.id'], '6e0c63257de34c90bf9efcd03927272e') @@ -153,10 +192,16 @@ def test_span_data_to_envelope(self): '2010-10-24T07:28:38.123456Z') self.assertEqual( envelope.data.baseData.name, + 'GET /wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.data, + 'https://www.wikipedia.org/wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.target, 'www.wikipedia.org') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') self.assertEqual( envelope.data.baseData.resultCode, '200') @@ -169,9 +214,20 @@ def test_span_data_to_envelope(self): self.assertEqual( envelope.data.baseType, 'RemoteDependencyData') + json_dict = json.loads( + envelope.data.baseData.properties["_MS.links"] + )[0] + self.assertEqual( + json_dict["id"], + "6e0c63257de34c91", + ) + self.assertEqual( + json_dict["operation_Id"], + "6e0c63257de34c90bf9efcd03927272e", + ) # SpanKind.CLIENT unknown type - envelope = exporter.span_data_to_envelope(SpanData( + envelope = next(exporter.span_data_to_envelope(SpanData( name='test', context=SpanContext( trace_id='6e0c63257de34c90bf9efcd03927272e', @@ -187,13 +243,13 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, child_span_count=None, span_kind=SpanKind.CLIENT, - )) + ))) self.assertEqual( envelope.iKey, '12345678-1234-5678-abcd-12345678abcd') @@ -202,7 +258,7 @@ def test_span_data_to_envelope(self): 'Microsoft.ApplicationInsights.RemoteDependency') self.assertEqual( envelope.tags['ai.operation.parentId'], - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c93.') + '6e0c63257de34c93') self.assertEqual( envelope.tags['ai.operation.id'], '6e0c63257de34c90bf9efcd03927272e') @@ -214,7 +270,75 @@ def test_span_data_to_envelope(self): 'test') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') + self.assertEqual( + envelope.data.baseData.duration, + '0.00:00:00.111') + self.assertEqual( + envelope.data.baseData.type, + None) + self.assertEqual( + envelope.data.baseType, + 'RemoteDependencyData') + + # SpanKind.CLIENT missing method + envelope = next(exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'component': 'HTTP', + 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', + 'http.status_code': 200, + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(0), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.CLIENT, + ))) + self.assertEqual( + envelope.iKey, + '12345678-1234-5678-abcd-12345678abcd') + self.assertEqual( + envelope.name, + 'Microsoft.ApplicationInsights.RemoteDependency') + self.assertEqual( + envelope.tags['ai.operation.parentId'], + '6e0c63257de34c93') + self.assertEqual( + envelope.tags['ai.operation.id'], + '6e0c63257de34c90bf9efcd03927272e') + self.assertEqual( + envelope.time, + '2010-10-24T07:28:38.123456Z') + self.assertEqual( + envelope.data.baseData.name, + 'test') + self.assertEqual( + envelope.data.baseData.data, + 'https://www.wikipedia.org/wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.target, + 'www.wikipedia.org') + self.assertEqual( + envelope.data.baseData.id, + '6e0c63257de34c92') + self.assertEqual( + envelope.data.baseData.resultCode, + '200') self.assertEqual( envelope.data.baseData.duration, '0.00:00:00.111') @@ -225,8 +349,8 @@ def test_span_data_to_envelope(self): envelope.data.baseType, 'RemoteDependencyData') - # SpanKind.SERVER HTTP - envelope = exporter.span_data_to_envelope(SpanData( + # SpanKind.SERVER HTTP - 200 request + envelope = next(exporter.span_data_to_envelope(SpanData( name='test', context=SpanContext( trace_id='6e0c63257de34c90bf9efcd03927272e', @@ -238,7 +362,10 @@ def test_span_data_to_envelope(self): span_id='6e0c63257de34c92', parent_span_id='6e0c63257de34c93', attributes={ + 'component': 'HTTP', 'http.method': 'GET', + 'http.path': '/wiki/Rabbit', + 'http.route': '/wiki/Rabbit', 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', 'http.status_code': 200, }, @@ -246,13 +373,13 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, child_span_count=None, span_kind=SpanKind.SERVER, - )) + ))) self.assertEqual( envelope.iKey, '12345678-1234-5678-abcd-12345678abcd') @@ -261,16 +388,19 @@ def test_span_data_to_envelope(self): 'Microsoft.ApplicationInsights.Request') self.assertEqual( envelope.tags['ai.operation.parentId'], - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c93.') + '6e0c63257de34c93') self.assertEqual( envelope.tags['ai.operation.id'], '6e0c63257de34c90bf9efcd03927272e') + self.assertEqual( + envelope.tags['ai.operation.name'], + 'GET /wiki/Rabbit') self.assertEqual( envelope.time, '2010-10-24T07:28:38.123456Z') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') self.assertEqual( envelope.data.baseData.duration, '0.00:00:00.111') @@ -279,7 +409,183 @@ def test_span_data_to_envelope(self): '200') self.assertEqual( envelope.data.baseData.name, - 'GET https://www.wikipedia.org/wiki/Rabbit') + 'GET /wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.properties['request.name'], + 'GET /wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.success, + True) + self.assertEqual( + envelope.data.baseData.url, + 'https://www.wikipedia.org/wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.properties['request.url'], + 'https://www.wikipedia.org/wiki/Rabbit') + self.assertEqual( + envelope.data.baseType, + 'RequestData') + + # SpanKind.SERVER HTTP - Failed request + envelope = next(exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'component': 'HTTP', + 'http.method': 'GET', + 'http.path': '/wiki/Rabbit', + 'http.route': '/wiki/Rabbit', + 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', + 'http.status_code': 400, + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(0), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + ))) + self.assertEqual( + envelope.iKey, + '12345678-1234-5678-abcd-12345678abcd') + self.assertEqual( + envelope.name, + 'Microsoft.ApplicationInsights.Request') + self.assertEqual( + envelope.tags['ai.operation.parentId'], + '6e0c63257de34c93') + self.assertEqual( + envelope.tags['ai.operation.id'], + '6e0c63257de34c90bf9efcd03927272e') + self.assertEqual( + envelope.tags['ai.operation.name'], + 'GET /wiki/Rabbit') + self.assertEqual( + envelope.time, + '2010-10-24T07:28:38.123456Z') + self.assertEqual( + envelope.data.baseData.id, + '6e0c63257de34c92') + self.assertEqual( + envelope.data.baseData.duration, + '0.00:00:00.111') + self.assertEqual( + envelope.data.baseData.responseCode, + '400') + self.assertEqual( + envelope.data.baseData.name, + 'GET /wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.success, + False) + self.assertEqual( + envelope.data.baseData.url, + 'https://www.wikipedia.org/wiki/Rabbit') + self.assertEqual( + envelope.data.baseType, + 'RequestData') + + # SpanKind.SERVER HTTP - with exceptions + envelopes = list(exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'component': 'HTTP', + 'http.method': 'GET', + 'http.path': '/wiki/Rabbit', + 'http.route': '/wiki/Rabbit', + 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', + 'http.status_code': 400, + 'error.message': 'bork bork', + 'error.name': 'ValueError', + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(0), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + ))) + self.assertEqual(len(envelopes), 2) + + envelope = envelopes[0] + self.assertEqual( + envelope.iKey, + '12345678-1234-5678-abcd-12345678abcd') + self.assertEqual( + envelope.name, + 'Microsoft.ApplicationInsights.Exception') + self.assertEqual( + envelope.tags['ai.operation.parentId'], + '6e0c63257de34c93') + self.assertEqual( + envelope.tags['ai.operation.id'], + '6e0c63257de34c90bf9efcd03927272e') + self.assertEqual( + envelope.time, + '2010-10-24T07:28:38.123456Z') + self.assertEqual( + envelope.data.baseType, + 'ExceptionData') + + envelope = envelopes[1] + self.assertEqual( + envelope.iKey, + '12345678-1234-5678-abcd-12345678abcd') + self.assertEqual( + envelope.name, + 'Microsoft.ApplicationInsights.Request') + self.assertEqual( + envelope.tags['ai.operation.parentId'], + '6e0c63257de34c93') + self.assertEqual( + envelope.tags['ai.operation.id'], + '6e0c63257de34c90bf9efcd03927272e') + self.assertEqual( + envelope.tags['ai.operation.name'], + 'GET /wiki/Rabbit') + self.assertEqual( + envelope.time, + '2010-10-24T07:28:38.123456Z') + self.assertEqual( + envelope.data.baseData.id, + '6e0c63257de34c92') + self.assertEqual( + envelope.data.baseData.duration, + '0.00:00:00.111') + self.assertEqual( + envelope.data.baseData.responseCode, + '400') + self.assertEqual( + envelope.data.baseData.name, + 'GET /wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.success, + False) self.assertEqual( envelope.data.baseData.url, 'https://www.wikipedia.org/wiki/Rabbit') @@ -288,7 +594,7 @@ def test_span_data_to_envelope(self): 'RequestData') # SpanKind.SERVER unknown type - envelope = exporter.span_data_to_envelope(SpanData( + envelope = next(exporter.span_data_to_envelope(SpanData( name='test', context=SpanContext( trace_id='6e0c63257de34c90bf9efcd03927272e', @@ -304,13 +610,13 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, child_span_count=None, span_kind=SpanKind.SERVER, - )) + ))) self.assertEqual( envelope.iKey, '12345678-1234-5678-abcd-12345678abcd') @@ -319,7 +625,7 @@ def test_span_data_to_envelope(self): 'Microsoft.ApplicationInsights.Request') self.assertEqual( envelope.tags['ai.operation.parentId'], - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c93.') + '6e0c63257de34c93') self.assertEqual( envelope.tags['ai.operation.id'], '6e0c63257de34c90bf9efcd03927272e') @@ -328,7 +634,7 @@ def test_span_data_to_envelope(self): '2010-10-24T07:28:38.123456Z') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') self.assertEqual( envelope.data.baseData.duration, '0.00:00:00.111') @@ -337,7 +643,7 @@ def test_span_data_to_envelope(self): 'RequestData') # SpanKind.UNSPECIFIED - envelope = exporter.span_data_to_envelope(SpanData( + envelope = next(exporter.span_data_to_envelope(SpanData( name='test', context=SpanContext( trace_id='6e0c63257de34c90bf9efcd03927272e', @@ -353,13 +659,13 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, child_span_count=None, span_kind=SpanKind.UNSPECIFIED, - )) + ))) self.assertEqual( envelope.iKey, '12345678-1234-5678-abcd-12345678abcd') @@ -383,192 +689,191 @@ def test_span_data_to_envelope(self): '0.00:00:00.111') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') self.assertEqual( envelope.data.baseData.type, 'INPROC') + self.assertEqual( + envelope.data.baseData.success, + True + ) self.assertEqual( envelope.data.baseType, 'RemoteDependencyData') - exporter._stop() - - def test_transmission_nothing(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - with mock.patch('requests.post') as post: - post.return_value = None - exporter._transmit_from_storage() - exporter._stop() - - def test_transmission_request_exception(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post', throw(Exception)): - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 1) - exporter._stop() - - @mock.patch('requests.post', return_value=mock.Mock()) - def test_transmission_lease_failure(self, requests_mock): - requests_mock.return_value = MockResponse(200, 'unknown') - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('opencensus.ext.azure.common.storage.LocalFileBlob.lease') as lease: # noqa: E501 - lease.return_value = False - exporter._transmit_from_storage() - self.assertTrue(exporter.storage.get()) - exporter._stop() - - def test_transmission_response_exception(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(200, None) - del post.return_value.text - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() - - def test_transmission_200(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(200, 'unknown') - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() + # Status server status code attribute + envelope = next(exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'http.status_code': 201 + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(0), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + ))) + self.assertEqual(envelope.data.baseData.responseCode, "201") + self.assertTrue(envelope.data.baseData.success) - def test_transmission_206(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(206, 'unknown') - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 1) - exporter._stop() + # Status server status code attribute missing + envelope = next(exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={}, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(1), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + ))) + self.assertFalse(envelope.data.baseData.success) - def test_transmission_206_500(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3, 4, 5]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(206, json.dumps({ - 'itemsReceived': 5, - 'itemsAccepted': 3, - 'errors': [ - { - 'index': 0, - 'statusCode': 400, - 'message': '', - }, - { - 'index': 2, - 'statusCode': 500, - 'message': 'Internal Server Error', - }, - ], - })) - exporter._transmit_from_storage() - self.assertEqual(len(os.listdir(exporter.storage.path)), 1) - self.assertEqual(exporter.storage.get().get(), (3,)) - exporter._stop() + # Server route attribute missing + envelope = next(exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'component': 'HTTP', + 'http.method': 'GET', + 'http.path': '/wiki/Rabbitz', + 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', + 'http.status_code': 400, + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(1), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + ))) + self.assertEqual(envelope.data.baseData.properties['request.name'], + 'GET /wiki/Rabbitz') - def test_transmission_206_nothing_to_retry(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(206, json.dumps({ - 'itemsReceived': 3, - 'itemsAccepted': 2, - 'errors': [ - { - 'index': 0, - 'statusCode': 400, - 'message': '', - }, - ], - })) - exporter._transmit_from_storage() - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() + # Server route and path attribute missing + envelope = next(exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'component': 'HTTP', + 'http.method': 'GET', + 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', + 'http.status_code': 400, + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(1), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + ))) + self.assertIsNone( + envelope.data.baseData.properties.get('request.name')) - def test_transmission_206_bogus(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3, 4, 5]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(206, json.dumps({ - 'itemsReceived': 5, - 'itemsAccepted': 3, - 'errors': [ - { - 'foo': 0, - 'bar': 1, - }, - ], - })) - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() + # Status client status code attribute + envelope = next(exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'http.status_code': 201 + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(0), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.CLIENT, + ))) + self.assertEqual(envelope.data.baseData.resultCode, "201") + self.assertTrue(envelope.data.baseData.success) - def test_transmission_400(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(400, '{}') - exporter._transmit_from_storage() - self.assertEqual(len(os.listdir(exporter.storage.path)), 0) - exporter._stop() + # Status client status code attributes missing + envelope = next(exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={}, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(1), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.CLIENT, + ))) + self.assertFalse(envelope.data.baseData.success) - def test_transmission_500(self): - exporter = trace_exporter.AzureExporter( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - storage_path=os.path.join(TEST_FOLDER, self.id()), - ) - exporter.storage.put([1, 2, 3]) - with mock.patch('requests.post') as post: - post.return_value = MockResponse(500, '{}') - exporter._transmit_from_storage() - self.assertIsNone(exporter.storage.get()) - self.assertEqual(len(os.listdir(exporter.storage.path)), 1) exporter._stop() - - -class MockResponse(object): - def __init__(self, status_code, text): - self.status_code = status_code - self.text = text diff --git a/contrib/opencensus-ext-azure/tests/test_azure_utils.py b/contrib/opencensus-ext-azure/tests/test_azure_utils.py index 79c7bfdb3..75aa0bb06 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_utils.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_utils.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest + from opencensus.ext.azure.common import utils @@ -36,9 +37,98 @@ def test_timestamp_to_iso_str(self): 1287905318.123456, ), '2010-10-24T07:28:38.123456Z') - def test_url_to_dependency_name(self): - self.assertEqual( - utils.url_to_dependency_name( - 'https://www.wikipedia.org/wiki/Rabbit' - ), - 'www.wikipedia.org') + def test_validate_instrumentation_key(self): + key = '1234abcd-5678-4efa-8abc-1234567890ab' + self.assertIsNone(utils.validate_instrumentation_key(key)) + + def test_invalid_key_none(self): + key = None + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_empty(self): + key = '' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_prefix(self): + key = 'test1234abcd-5678-4efa-8abc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_suffix(self): + key = '1234abcd-5678-4efa-8abc-1234567890abtest' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_length(self): + key = '1234abcd-5678-4efa-8abc-12234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_dashes(self): + key = '1234abcda5678-4efa-8abc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section1_length(self): + key = '1234abcda-678-4efa-8abc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section2_length(self): + key = '1234abcd-678-a4efa-8abc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section3_length(self): + key = '1234abcd-6789-4ef-8cabc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section4_length(self): + key = '1234abcd-678-4efa-8bc-11234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section5_length(self): + key = '234abcd-678-4efa-8abc-11234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section1_hex(self): + key = 'x234abcd-5678-4efa-8abc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section2_hex(self): + key = '1234abcd-x678-4efa-8abc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section3_hex(self): + key = '1234abcd-5678-4xfa-8abc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section4_hex(self): + key = '1234abcd-5678-4xfa-8abc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_invalid_key_section5_hex(self): + key = '1234abcd-5678-4xfa-8abc-1234567890ab' + self.assertRaises(ValueError, + lambda: utils.validate_instrumentation_key(key)) + + def test_valid_key_section2_hex(self): + key = '1234abcd-567a-4efa-8abc-1234567890ab' + self.assertIsNone(utils.validate_instrumentation_key(key)) + + def test_valid_key_section3_hex(self): + key = '1234abcd-5678-befa-8abc-1234567890ab' + self.assertIsNone(utils.validate_instrumentation_key(key)) + + def test_valid_key_section4_hex(self): + key = '1234abcd-5678-4efa-cabc-1234567890ab' + self.assertIsNone(utils.validate_instrumentation_key(key)) diff --git a/contrib/opencensus-ext-azure/tests/test_options.py b/contrib/opencensus-ext-azure/tests/test_options.py new file mode 100644 index 000000000..9ab130fb5 --- /dev/null +++ b/contrib/opencensus-ext-azure/tests/test_options.py @@ -0,0 +1,156 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest + +from opencensus.ext.azure import common + + +class TestOptions(unittest.TestCase): + def setUp(self): + os.environ.clear() + + def test_process_options_ikey_code_cs(self): + options = common.Options() + options.connection_string = 'Authorization=ikey;InstrumentationKey=123' + options.instrumentation_key = '456' + os.environ['APPLICATIONINSIGHTS_CONNECTION_STRING'] = \ + 'Authorization=ikey;InstrumentationKey=789' + os.environ['APPINSIGHTS_INSTRUMENTATIONKEY'] = '101112' + common.process_options(options) + + self.assertEqual(options.instrumentation_key, '123') + + def test_process_options_ikey_code_ikey(self): + options = common.Options() + options.connection_string = None + options.instrumentation_key = '456' + os.environ['APPLICATIONINSIGHTS_CONNECTION_STRING'] = \ + 'Authorization=ikey;InstrumentationKey=789' + os.environ['APPINSIGHTS_INSTRUMENTATIONKEY'] = '101112' + common.process_options(options) + + self.assertEqual(options.instrumentation_key, '456') + + def test_process_options_ikey_env_cs(self): + options = common.Options() + options.connection_string = None + options.instrumentation_key = None + os.environ['APPLICATIONINSIGHTS_CONNECTION_STRING'] = \ + 'Authorization=ikey;InstrumentationKey=789' + os.environ['APPINSIGHTS_INSTRUMENTATIONKEY'] = '101112' + common.process_options(options) + + self.assertEqual(options.instrumentation_key, '789') + + def test_process_options_ikey_env_ikey(self): + options = common.Options() + options.connection_string = None + options.instrumentation_key = None + os.environ['APPINSIGHTS_INSTRUMENTATIONKEY'] = '101112' + common.process_options(options) + + self.assertEqual(options.instrumentation_key, '101112') + + def test_process_options_endpoint_code_cs(self): + options = common.Options() + options.connection_string = 'Authorization=ikey;IngestionEndpoint=123' + os.environ['APPLICATIONINSIGHTS_CONNECTION_STRING'] = \ + 'Authorization=ikey;IngestionEndpoint=456' + common.process_options(options) + + self.assertEqual(options.endpoint, '123') + + def test_process_options_endpoint_env_cs(self): + options = common.Options() + options.connection_string = None + os.environ['APPLICATIONINSIGHTS_CONNECTION_STRING'] = \ + 'Authorization=ikey;IngestionEndpoint=456' + common.process_options(options) + + self.assertEqual(options.endpoint, '456') + + def test_process_options_endpoint_default(self): + options = common.Options() + options.connection_string = None + common.process_options(options) + + self.assertEqual(options.endpoint, + 'https://dc.services.visualstudio.com') + + def test_process_options_proxies_default(self): + options = common.Options() + options.proxies = "{}" + common.process_options(options) + + self.assertEqual(options.proxies, "{}") + + def test_process_options_proxies_set_proxies(self): + options = common.Options() + options.connection_string = None + options.proxies = '{"https": "https://test-proxy.com"}' + common.process_options(options) + + self.assertEqual( + options.proxies, + '{"https": "https://test-proxy.com"}' + ) + + def test_parse_connection_string_none(self): + cs = None + result = common.parse_connection_string(cs) + + self.assertEqual(result, {}) + + def test_parse_connection_string_invalid(self): + cs = 'asd' + self.assertRaises(ValueError, + lambda: common.parse_connection_string(cs)) + + def test_parse_connection_string_default_auth(self): + cs = 'InstrumentationKey=123' + result = common.parse_connection_string(cs) + self.assertEqual(result['instrumentationkey'], '123') + + def test_parse_connection_string_invalid_auth(self): + cs = 'Authorization=asd' + self.assertRaises(ValueError, + lambda: common.parse_connection_string(cs)) + + def test_parse_connection_string_explicit_endpoint(self): + cs = 'Authorization=ikey;IngestionEndpoint=123;' \ + 'Location=us;EndpointSuffix=suffix' + result = common.parse_connection_string(cs) + + self.assertEqual(result['ingestionendpoint'], '123') + + def test_parse_connection_string_default(self): + cs = 'Authorization=ikey;Location=us' + result = common.parse_connection_string(cs) + + self.assertEqual(result['ingestionendpoint'], + None) + + def test_parse_connection_string_no_location(self): + cs = 'Authorization=ikey;EndpointSuffix=suffix' + result = common.parse_connection_string(cs) + + self.assertEqual(result['ingestionendpoint'], 'https://dc.suffix') + + def test_parse_connection_string_location(self): + cs = 'Authorization=ikey;EndpointSuffix=suffix;Location=us' + result = common.parse_connection_string(cs) + + self.assertEqual(result['ingestionendpoint'], 'https://us.dc.suffix') diff --git a/contrib/opencensus-ext-azure/tests/test_processor_mixin.py b/contrib/opencensus-ext-azure/tests/test_processor_mixin.py new file mode 100644 index 000000000..7ec01eb83 --- /dev/null +++ b/contrib/opencensus-ext-azure/tests/test_processor_mixin.py @@ -0,0 +1,94 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from opencensus.ext.azure.common.processor import ProcessorMixin +from opencensus.ext.azure.common.protocol import Envelope + + +# pylint: disable=W0212 +class TestProcessorMixin(unittest.TestCase): + def test_add(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + mixin.add_telemetry_processor(lambda: True) + self.assertEqual(len(mixin._telemetry_processors), 1) + + def test_clear(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + mixin.add_telemetry_processor(lambda: True) + self.assertEqual(len(mixin._telemetry_processors), 1) + mixin.clear_telemetry_processors() + self.assertEqual(len(mixin._telemetry_processors), 0) + + def test_apply(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + + def callback_function(envelope): + envelope.baseType += '_world' + mixin.add_telemetry_processor(callback_function) + envelope = Envelope() + envelope.baseType = 'type1' + mixin.apply_telemetry_processors([envelope]) + self.assertEqual(envelope.baseType, 'type1_world') + + def test_apply_multiple(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + + def callback_function(envelope): + envelope.baseType += '_world' + + def callback_function2(envelope): + envelope.baseType += '_world2' + mixin.add_telemetry_processor(callback_function) + mixin.add_telemetry_processor(callback_function2) + envelope = Envelope() + envelope.baseType = 'type1' + mixin.apply_telemetry_processors([envelope]) + self.assertEqual(envelope.baseType, 'type1_world_world2') + + def test_apply_exception(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + + def callback_function(envelope): + raise ValueError() + + def callback_function2(envelope): + envelope.baseType += '_world2' + mixin.add_telemetry_processor(callback_function) + mixin.add_telemetry_processor(callback_function2) + envelope = Envelope() + envelope.baseType = 'type1' + mixin.apply_telemetry_processors([envelope]) + self.assertEqual(envelope.baseType, 'type1_world2') + + def test_apply_not_accepted(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + + def callback_function(envelope): + return envelope.baseType == 'type2' + mixin.add_telemetry_processor(callback_function) + envelope = Envelope() + envelope.baseType = 'type1' + envelope2 = Envelope() + envelope2.baseType = 'type2' + envelopes = mixin.apply_telemetry_processors([envelope, envelope2]) + self.assertEqual(len(envelopes), 1) + self.assertEqual(envelopes[0].baseType, 'type2') diff --git a/contrib/opencensus-ext-azure/tests/test_protocol.py b/contrib/opencensus-ext-azure/tests/test_protocol.py index 7d7a9f2cd..dcfc830e6 100644 --- a/contrib/opencensus-ext-azure/tests/test_protocol.py +++ b/contrib/opencensus-ext-azure/tests/test_protocol.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest + from opencensus.ext.azure.common import protocol diff --git a/contrib/opencensus-ext-azure/tests/test_storage.py b/contrib/opencensus-ext-azure/tests/test_storage.py index 9b9b2e12b..62b12dea3 100644 --- a/contrib/opencensus-ext-azure/tests/test_storage.py +++ b/contrib/opencensus-ext-azure/tests/test_storage.py @@ -12,17 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock import os import shutil import unittest -from opencensus.ext.azure.common.storage import _now -from opencensus.ext.azure.common.storage import _seconds -from opencensus.ext.azure.common.storage import LocalFileBlob -from opencensus.ext.azure.common.storage import LocalFileStorage +import mock + +from opencensus.ext.azure.common.storage import ( + LocalFileBlob, + LocalFileStorage, + _now, + _seconds, +) -TEST_FOLDER = os.path.abspath('.test') +TEST_FOLDER = os.path.abspath('.test.storage') def setUpModule(): @@ -42,39 +45,33 @@ def func(*_args, **_kwargs): class TestLocalFileBlob(unittest.TestCase): def test_delete(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar')) - blob.delete(silent=True) - self.assertRaises(Exception, lambda: blob.delete()) - self.assertRaises(Exception, lambda: blob.delete(silent=False)) + blob.delete() + with mock.patch('os.remove') as m: + blob.delete() + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'foobar')) def test_get(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar')) - self.assertIsNone(blob.get(silent=True)) - self.assertRaises(Exception, lambda: blob.get()) - self.assertRaises(Exception, lambda: blob.get(silent=False)) - - def test_put_error(self): - blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar')) - with mock.patch('os.rename', side_effect=throw(Exception)): - self.assertRaises(Exception, lambda: blob.put([1, 2, 3])) + self.assertIsNone(blob.get()) def test_put_without_lease(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar.blob')) input = (1, 2, 3) - blob.delete(silent=True) + blob.delete() blob.put(input) self.assertEqual(blob.get(), input) def test_put_with_lease(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar.blob')) input = (1, 2, 3) - blob.delete(silent=True) + blob.delete() blob.put(input, lease_period=0.01) blob.lease(0.01) self.assertEqual(blob.get(), input) def test_lease_error(self): blob = LocalFileBlob(os.path.join(TEST_FOLDER, 'foobar.blob')) - blob.delete(silent=True) + blob.delete() self.assertEqual(blob.lease(0.01), None) @@ -110,36 +107,81 @@ def test_put(self): with LocalFileStorage(os.path.join(TEST_FOLDER, 'bar')) as stor: self.assertEqual(stor.get().get(), input) with mock.patch('os.rename', side_effect=throw(Exception)): - self.assertIsNone(stor.put(input, silent=True)) - self.assertRaises(Exception, lambda: stor.put(input)) + self.assertIsNone(stor.put(input)) + + def test_put_max_size(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd')) as stor: + size_mock = mock.Mock() + size_mock.return_value = False + stor._check_storage_size = size_mock + stor.put(input) + self.assertEqual(stor.get(), None) + + def test_check_storage_size_full(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd2'), 1) as stor: + stor.put(input) + self.assertFalse(stor._check_storage_size()) - def test_maintanence_routine(self): + def test_check_storage_size_not_full(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor: + stor.put(input) + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_no_files(self): + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd3'), 1000) as stor: + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_links(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd4'), 1000) as stor: + stor.put(input) + with mock.patch('os.path.islink') as os_mock: + os_mock.return_value = True + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_error(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd5'), 1) as stor: + with mock.patch('os.path.getsize', side_effect=throw(OSError)): + stor.put(input) + with mock.patch('os.path.islink') as os_mock: + os_mock.return_value = True + self.assertTrue(stor._check_storage_size()) + + def test_check_storage_size_above_max_limit(self): + input = (1, 2, 3) + with LocalFileStorage(os.path.join(TEST_FOLDER, 'asd5'), 1) as stor: + with mock.patch('os.path.getsize') as os_mock: + os_mock.return_value = 52000000 + stor.put(input) + with mock.patch('os.path.islink') as os_mock: + os_mock.return_value = True + self.assertFalse(stor._check_storage_size()) + + def test_maintenance_routine(self): + with mock.patch('os.makedirs') as m: + LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'baz')) with mock.patch('os.makedirs') as m: m.return_value = None - self.assertRaises( - Exception, - lambda: LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')), - ) + LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'baz')) with mock.patch('os.makedirs', side_effect=throw(Exception)): - self.assertRaises( - Exception, - lambda: LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')), - ) + LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'baz')) with mock.patch('os.listdir', side_effect=throw(Exception)): - self.assertRaises( - Exception, - lambda: LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')), - ) + LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) + m.assert_called_once_with(os.path.join(TEST_FOLDER, 'baz')) with LocalFileStorage(os.path.join(TEST_FOLDER, 'baz')) as stor: - with mock.patch('os.listdir', side_effect=throw(Exception)): - stor._maintenance_routine(silent=True) - self.assertRaises( - Exception, - lambda: stor._maintenance_routine(), - ) - with mock.patch('os.path.isdir', side_effect=throw(Exception)): - stor._maintenance_routine(silent=True) - self.assertRaises( - Exception, - lambda: stor._maintenance_routine(), - ) + with mock.patch('os.listdir', side_effect=throw(Exception)) as p: + stor._maintenance_routine() + stor._maintenance_routine() + self.assertEqual(p.call_count, 2) + patch = 'os.path.isdir' + with mock.patch(patch, side_effect=throw(Exception)) as isdir: + stor._maintenance_routine() + stor._maintenance_routine() + self.assertEqual(isdir.call_count, 2) diff --git a/contrib/opencensus-ext-azure/tests/test_transport_mixin.py b/contrib/opencensus-ext-azure/tests/test_transport_mixin.py new file mode 100644 index 000000000..f955adfc5 --- /dev/null +++ b/contrib/opencensus-ext-azure/tests/test_transport_mixin.py @@ -0,0 +1,783 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import shutil +import unittest + +import mock +import requests +from azure.core.exceptions import ClientAuthenticationError +from azure.identity._exceptions import CredentialUnavailableError + +from opencensus.ext.azure.common import Options +from opencensus.ext.azure.common.storage import LocalFileStorage +from opencensus.ext.azure.common.transport import ( + _MAX_CONSECUTIVE_REDIRECTS, + _MONITOR_OAUTH_SCOPE, + _REACHED_INGESTION_STATUS_CODES, + TransportMixin, + TransportStatusCode, + _requests_map, +) +from opencensus.ext.azure.statsbeat import state + +TEST_FOLDER = os.path.abspath('.test.transport') + + +def setUpModule(): + os.makedirs(TEST_FOLDER) + + +def tearDownModule(): + shutil.rmtree(TEST_FOLDER) + + +def throw(exc_type, *args, **kwargs): + def func(*_args, **_kwargs): + raise exc_type(*args, **kwargs) + return func + + +class MockResponse(object): + def __init__(self, status_code, text, headers=None): + self.status_code = status_code + self.text = text + self.headers = headers + + +# pylint: disable=W0212 +class TestTransportMixin(unittest.TestCase): + def setUp(self): + # pylint: disable=protected-access + _requests_map.clear() + state._STATSBEAT_STATE = { + "INITIAL_FAILURE_COUNT": 0, + "INITIAL_SUCCESS": False, + "SHUTDOWN": False, + } + + def test_check_stats_collection(self): + mixin = TransportMixin() + with mock.patch.dict( + os.environ, { + "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "", + }): + self.assertTrue(mixin._check_stats_collection()) + with mock.patch.dict( + os.environ, { + "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "True", + }): + self.assertFalse(mixin._check_stats_collection()) + mixin._is_stats = False + self.assertTrue(mixin._check_stats_collection()) + mixin._is_stats = True + self.assertFalse(mixin._check_stats_collection()) + mixin._is_stats = False + state._STATSBEAT_STATE["SHUTDOWN"] = False + self.assertTrue(mixin._check_stats_collection()) + state._STATSBEAT_STATE["SHUTDOWN"] = True + self.assertFalse(mixin._check_stats_collection()) + + def test_initial_statsbeat_success(self): + self.assertFalse(state._STATSBEAT_STATE["INITIAL_SUCCESS"]) + mixin = TransportMixin() + mixin.options = Options() + mixin._is_stats = True + with mock.patch('requests.post') as post: + for code in _REACHED_INGESTION_STATUS_CODES: + post.return_value = MockResponse(code, 'unknown') + mixin._transmit([1]) + self.assertTrue(state._STATSBEAT_STATE["INITIAL_SUCCESS"]) + state._STATSBEAT_STATE["INITIAL_SUCCESS"] = False + + def test_exception_statsbeat_shutdown_increment(self): + mixin = TransportMixin() + mixin.options = Options() + mixin._is_stats = True + state._STATSBEAT_STATE["INITIAL_SUCCESS"] = False + state._STATSBEAT_STATE["INITIAL_FAILURE_COUNT"] = 0 + state._STATSBEAT_STATE["SHUTDOWN"] = False + with mock.patch.dict( + os.environ, { + "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "", + }): + with mock.patch('requests.post', throw(Exception)): + result = mixin._transmit([1, 2, 3]) + self.assertEqual(state._STATSBEAT_STATE["INITIAL_FAILURE_COUNT"], 1) # noqa: E501 + self.assertEqual(result, TransportStatusCode.DROP) + + def test_exception_statsbeat_shutdown(self): + mixin = TransportMixin() + mixin.options = Options() + mixin._is_stats = True + state._STATSBEAT_STATE["INITIAL_SUCCESS"] = False + state._STATSBEAT_STATE["INITIAL_FAILURE_COUNT"] = 2 + state._STATSBEAT_STATE["SHUTDOWN"] = False + with mock.patch.dict( + os.environ, { + "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "", + }): + with mock.patch('requests.post', throw(Exception)): + result = mixin._transmit([1, 2, 3]) + self.assertEqual(state._STATSBEAT_STATE["INITIAL_FAILURE_COUNT"], 3) # noqa: E501 + self.assertEqual(result, TransportStatusCode.STATSBEAT_SHUTDOWN) # noqa: E501 + + def test_status_code_statsbeat_shutdown_increment(self): + mixin = TransportMixin() + mixin.options = Options() + mixin._is_stats = True + state._STATSBEAT_STATE["INITIAL_SUCCESS"] = False + state._STATSBEAT_STATE["INITIAL_FAILURE_COUNT"] = 0 + state._STATSBEAT_STATE["SHUTDOWN"] = False + with mock.patch.dict( + os.environ, { + "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "", + }): + with mock.patch('requests.post') as post: + post.return_value = MockResponse(403, 'unknown') + mixin._transmit([1, 2, 3]) + self.assertEqual(state._STATSBEAT_STATE["INITIAL_FAILURE_COUNT"], 1) # noqa: E501 + self.assertFalse(state._STATSBEAT_STATE["INITIAL_SUCCESS"]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(200, 'unknown') + mixin._transmit([1, 2, 3]) + self.assertEqual(state._STATSBEAT_STATE["INITIAL_FAILURE_COUNT"], 1) # noqa: E501 + self.assertTrue(state._STATSBEAT_STATE["INITIAL_SUCCESS"]) + + def test_status_code_statsbeat_shutdown(self): + mixin = TransportMixin() + mixin.options = Options() + mixin._is_stats = True + state._STATSBEAT_STATE["INITIAL_SUCCESS"] = False + state._STATSBEAT_STATE["INITIAL_FAILURE_COUNT"] = 2 + state._STATSBEAT_STATE["SHUTDOWN"] = False + with mock.patch.dict( + os.environ, { + "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "", + }): + with mock.patch('requests.post') as post: + post.return_value = MockResponse(403, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(state._STATSBEAT_STATE["INITIAL_FAILURE_COUNT"], 3) # noqa: E501 + self.assertFalse(state._STATSBEAT_STATE["INITIAL_SUCCESS"]) + self.assertEqual(result, TransportStatusCode.STATSBEAT_SHUTDOWN) # noqa: E501 + + def test_transmission_nothing(self): + mixin = TransportMixin() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + with mock.patch('requests.post') as post: + post.return_value = None + mixin._transmit_from_storage() + + def test_transmission_timeout(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post', throw(requests.Timeout)): + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_timeout(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post', throw(requests.Timeout)): + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(_requests_map['exception']['Timeout'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_req_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post', throw(requests.RequestException)): + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_req_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post', throw(requests.RequestException)): + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(_requests_map['exception']['RequestException'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_cred_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post', throw(CredentialUnavailableError)): # noqa: E501 + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_statsbeat_cred_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post', throw(CredentialUnavailableError)): + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['exception']['CredentialUnavailableError'], 1) # noqa: E501 + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.DROP) + + def test_transmission_client_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post', throw(ClientAuthenticationError)): + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_client_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post', throw(ClientAuthenticationError)): + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['exception']['ClientAuthenticationError'], 1) # noqa: E501 + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post', throw(Exception)): + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_statsbeat_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post', throw(Exception)): + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['exception']['Exception'], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.DROP) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_transmission_lease_failure(self, requests_mock): + requests_mock.return_value = MockResponse(200, 'unknown') + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch( + 'opencensus.ext.azure.common.storage.LocalFileBlob.lease' + ) as lease: # noqa: E501 + lease.return_value = False + mixin._transmit_from_storage() + self.assertTrue(mixin.storage.get()) + + def test_transmission_text_exception(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(200, None) + del post.return_value.text + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_transmission_200(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(200, 'unknown') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_statsbeat_200(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(200, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['success'], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.SUCCESS) + + def test_transmission_auth(self): + mixin = TransportMixin() + mixin.options = Options() + url = 'https://dc.services.visualstudio.com' + mixin.options.endpoint = url + credential = mock.Mock() + mixin.options.credential = credential + token_mock = mock.Mock() + token_mock.token = "test_token" + credential.get_token.return_value = token_mock + data = '[1, 2, 3]' + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json; charset=utf-8', + 'Authorization': 'Bearer test_token', + } + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(200, 'unknown') + mixin._transmit_from_storage() + post.assert_called_with( + url=url + '/v2.1/track', + data=data, + headers=headers, + timeout=10.0, + proxies={}, + allow_redirects=False, + ) + credential.get_token.assert_called_with(_MONITOR_OAUTH_SCOPE) + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + credential.get_token.assert_called_once() + + def test_transmission_206_invalid_data(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, 'unknown') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_statsbeat_206_invalid_data(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.DROP) + + def test_transmission_206_partial_retry(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3, 4, 5]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, json.dumps({ + 'itemsReceived': 5, + 'itemsAccepted': 3, + 'errors': [ + { + 'index': 0, + 'statusCode': 400, # dropped + 'message': '', + }, + { + 'index': 2, + 'statusCode': 500, # retry + 'message': 'Internal Server Error', + }, + ], + })) + mixin._transmit_from_storage() + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + self.assertEqual(mixin.storage.get().get(), (3,)) + + def test_statsbeat_206_partial_retry(self): + mixin = TransportMixin() + mixin.options = Options() + storage_mock = mock.Mock() + mixin.storage = storage_mock + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, json.dumps({ + 'itemsReceived': 5, + 'itemsAccepted': 3, + 'errors': [ + { + 'index': 0, + 'statusCode': 400, # dropped + 'message': '', + }, + { + 'index': 2, + 'statusCode': 500, # retry + 'message': 'Internal Server Error', + }, + ], + })) + result = mixin._transmit([1, 2, 3]) + # We do not record any network statsbeat for 206 status code + self.assertEqual(len(_requests_map), 2) + self.assertIsNotNone(_requests_map['duration']) + self.assertIsNone(_requests_map.get('retry')) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.DROP) + storage_mock.put.assert_called_once() + + def test_transmission_206_no_retry(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, json.dumps({ + 'itemsReceived': 3, + 'itemsAccepted': 2, + 'errors': [ + { + 'index': 0, + 'statusCode': 400, # dropped + 'message': '', + }, + ], + })) + mixin._transmit_from_storage() + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_statsbeat_206_no_retry(self): + mixin = TransportMixin() + mixin.options = Options() + storage_mock = mock.Mock() + mixin.storage = storage_mock + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, json.dumps({ + 'itemsReceived': 3, + 'itemsAccepted': 2, + 'errors': [ + { + 'index': 0, + 'statusCode': 400, # dropped + 'message': '', + }, + ], + })) + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 2) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.DROP) + storage_mock.put.assert_not_called() + + def test_transmission_206_bogus(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3, 4, 5]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, json.dumps({ + 'itemsReceived': 5, + 'itemsAccepted': 3, + 'errors': [ + { + 'foo': 0, + 'bar': 1, + }, + ], + })) + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_statsbeat_206_bogus(self): + mixin = TransportMixin() + mixin.options = Options() + storage_mock = mock.Mock() + mixin.storage = storage_mock + with mock.patch('requests.post') as post: + post.return_value = MockResponse(206, json.dumps({ + 'itemsReceived': 5, + 'itemsAccepted': 3, + 'errors': [ + { + 'foo': 0, + 'bar': 1, + }, + ], + })) + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(_requests_map['exception']['KeyError'], 1) + self.assertEqual(result, TransportStatusCode.DROP) + storage_mock.put.assert_not_called() + + def test_transmission_429(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(429, 'unknown') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_429(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(429, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['retry'][429], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_500(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(500, 'unknown') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_500(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(500, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['retry'][500], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_502(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(502, 'unknown') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_502(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(502, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['retry'][502], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_503(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(503, 'unknown') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_503(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(503, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['retry'][503], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_504(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(504, 'unknown') + mixin._transmit_from_storage() + self.assertIsNone(mixin.storage.get()) + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_504(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(504, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['retry'][504], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_401(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(401, '{}') + mixin._transmit_from_storage() + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_401(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(401, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['retry'][401], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_403(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(403, '{}') + mixin._transmit_from_storage() + self.assertEqual(len(os.listdir(mixin.storage.path)), 1) + + def test_statsbeat_403(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(403, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['retry'][403], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.RETRY) + + def test_transmission_307(self): + mixin = TransportMixin() + mixin.options = Options() + mixin._consecutive_redirects = 0 + mixin.options.endpoint = "test.endpoint" + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(307, '{}', {"location": "https://example.com"}) # noqa: E501 + mixin._transmit_from_storage() + self.assertEqual(post.call_count, _MAX_CONSECUTIVE_REDIRECTS) + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + self.assertEqual(mixin.options.endpoint, "https://example.com") + + def test_transmission_307_circular_reference(self): + mixin = TransportMixin() + mixin.options = Options() + mixin._consecutive_redirects = 0 + mixin.options.endpoint = "https://example.com" + with mock.patch('requests.post') as post: + post.return_value = MockResponse(307, '{}', {"location": "https://example.com"}) # noqa: E501 + result = mixin._transmit([1, 2, 3]) + self.assertEqual(result, TransportStatusCode.DROP) + self.assertEqual(post.call_count, _MAX_CONSECUTIVE_REDIRECTS) + self.assertEqual(mixin.options.endpoint, "https://example.com") + + def test_statsbeat_307(self): + mixin = TransportMixin() + mixin.options = Options() + mixin._consecutive_redirects = 0 + mixin.options.endpoint = "test.endpoint" + with mock.patch('requests.post') as post: + post.return_value = MockResponse(307, '{}', {"location": "https://example.com"}) # noqa: E501 + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['exception']['Circular Redirect'], 1) # noqa: E501 + self.assertEqual(_requests_map['count'], 10) + self.assertEqual(result, TransportStatusCode.DROP) + + def test_transmission_439(self): + mixin = TransportMixin() + mixin.options = Options() + with LocalFileStorage(os.path.join(TEST_FOLDER, self.id())) as stor: + mixin.storage = stor + mixin.storage.put([1, 2, 3]) + with mock.patch('requests.post') as post: + post.return_value = MockResponse(439, '{}') + mixin._transmit_from_storage() + self.assertEqual(len(os.listdir(mixin.storage.path)), 0) + + def test_statsbeat_439(self): + mixin = TransportMixin() + mixin.options = Options() + with mock.patch('requests.post') as post: + post.return_value = MockResponse(439, 'unknown') + result = mixin._transmit([1, 2, 3]) + self.assertEqual(len(_requests_map), 3) + self.assertIsNotNone(_requests_map['duration']) + self.assertEqual(_requests_map['throttle'][439], 1) + self.assertEqual(_requests_map['count'], 1) + self.assertEqual(result, TransportStatusCode.DROP) diff --git a/contrib/opencensus-ext-datadog/CHANGELOG.md b/contrib/opencensus-ext-datadog/CHANGELOG.md new file mode 100644 index 000000000..c6bcf394c --- /dev/null +++ b/contrib/opencensus-ext-datadog/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## Unreleased + +## 0.1.0 +Released 2019-11-26 + +- Initial version + ([#793](https://github.com/census-instrumentation/opencensus-python/pull/793)) diff --git a/contrib/opencensus-ext-datadog/README.rst b/contrib/opencensus-ext-datadog/README.rst new file mode 100644 index 000000000..1fdcf158e --- /dev/null +++ b/contrib/opencensus-ext-datadog/README.rst @@ -0,0 +1,80 @@ +OpenCensus Datadog Exporter +============================================================================ + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opencensus-ext-datadog.svg + :target: https://pypi.org/project/opencensus-ext-datadog/ + +Installation +------------ + +:: + + pip install opencensus-ext-datadog + +Usage +----- + +Trace +~~~~~ + +The **Datadog Trace Exporter** allows you to export `OpenCensus`_ traces to `Datadog`_. + +This example shows how to send a span "hello" to Datadog. + +* Set up a `Datadog Agent `_ that is accessible to your app. +* Place the URL for the agent in the `trace_addr` of the configuration options. + + .. code:: python + + from opencensus.ext.datadog.traces import DatadogTraceExporter, Options + from opencensus.trace.samplers import ProbabilitySampler + from opencensus.trace.tracer import Tracer + + tracer = Tracer( + exporter=DatadogTraceExporter(Options(service='app-name',trace_addr='my-datdog-agent:8126`)), + sampler=ProbabilitySampler(1.0) + ) + + with tracer.span(name='hello'): + print('Hello, World!') + +OpenCensus also supports several `integrations `_ which allows OpenCensus to integrate with third party libraries. + +This example shows how to integrate with the `requests `_ library. + +* Set up a `Datadog Agent `_ that is accessible to your app. +* Place the URL for the agent in the `trace_addr` of the configuration options. + +.. code:: python + + import requests + + from opencensus.ext.datadog.traces import DatadogTraceExporter, Options + from opencensus.trace.samplers import ProbabilitySampler + from opencensus.trace.tracer import Tracer + + config_integration.trace_integrations(['requests']) + tracer = Tracer( + exporter=DatadogTraceExporter( + Options( + service='app-name', + trace_addr='my-datdog-agent:8126` + ) + ), + sampler=ProbabilitySampler(1.0), + ) + with tracer.span(name='parent'): + response = requests.get(url='https://www.wikipedia.org/wiki/Rabbit') + + +References +---------- + +* `Datadog `_ +* `Examples `_ +* `OpenCensus Project `_ + +.. _Datadog: https://www.datadoghq.com/product/ +.. _OpenCensus: https://github.com/census-instrumentation/opencensus-python/ diff --git a/contrib/opencensus-ext-datadog/examples/datadog.py b/contrib/opencensus-ext-datadog/examples/datadog.py new file mode 100644 index 000000000..a74e67c44 --- /dev/null +++ b/contrib/opencensus-ext-datadog/examples/datadog.py @@ -0,0 +1,22 @@ +from flask import Flask + +from opencensus.ext.datadog.traces import DatadogTraceExporter, Options +from opencensus.ext.flask.flask_middleware import FlaskMiddleware +from opencensus.trace.samplers import AlwaysOnSampler + +app = Flask(__name__) +middleware = FlaskMiddleware(app, + excludelist_paths=['/healthz'], + sampler=AlwaysOnSampler(), + exporter=DatadogTraceExporter( + Options(service='python-export-test', + global_tags={"stack": "example"}))) + + +@app.route('/') +def hello(): + return 'Hello World!' + + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=8080, threaded=True) diff --git a/contrib/opencensus-ext-datadog/opencensus/__init__.py b/contrib/opencensus-ext-datadog/opencensus/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/__init__.py b/contrib/opencensus-ext-datadog/opencensus/ext/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/ext/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/__init__.py b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py new file mode 100644 index 000000000..6555a801c --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/traces.py @@ -0,0 +1,372 @@ +# Copyright 2018, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import codecs +from collections import defaultdict +from datetime import datetime + +import bitarray + +from opencensus.common.transports import sync +from opencensus.common.utils import ISO_DATETIME_REGEX +from opencensus.ext.datadog.transport import DDTransport +from opencensus.trace import base_exporter, span_data + + +class Options(object): + """ Options contains options for configuring the exporter. + The address can be empty as the prometheus client will + assume it's localhost + + :type namespace: str + :param namespace: Namespace specifies the namespaces to which metric keys + are appended. Defaults to ''. + + :type service: str + :param service: service specifies the service name used for tracing. + + :type trace_addr: str + :param trace_addr: trace_addr specifies the host[:port] address of the + Datadog Trace Agent. It defaults to localhost:8126 + + :type global_tags: dict + :param global_tags: global_tags is a set of tags that will be + applied to all exported spans. + """ + def __init__(self, service='', trace_addr='localhost:8126', + global_tags={}): + self._service = service + self._trace_addr = trace_addr + for k, v in global_tags.items(): + if not isinstance(k, str) or not isinstance(v, str): + raise TypeError( + "global tags must be dictionary of string string") + self._global_tags = global_tags + + @property + def trace_addr(self): + """ specifies the host[:port] address of the Datadog Trace Agent. + """ + return self._trace_addr + + @property + def service(self): + """ Specifies the service name used for tracing. + """ + return self._service + + @property + def global_tags(self): + """ Specifies the namespaces to which metric keys are appended + """ + return self._global_tags + + +class DatadogTraceExporter(base_exporter.Exporter): + """ A exporter that send traces and trace spans to Datadog. + + :type options: :class:`~opencensus.ext.datadog.Options` + :param options: An options object with the parameters to instantiate the + Datadog Exporter. + + :type transport: + :class:`opencensus.common.transports.sync.SyncTransport` or + :class:`opencensus.common.transports.async_.AsyncTransport` + :param transport: An instance of a Transport to send data with. + """ + def __init__(self, options, transport=sync.SyncTransport): + self._options = options + self._transport = transport(self) + self._dd_transport = DDTransport(options.trace_addr) + + @property + def transport(self): + """ The transport way to be sent data to server + (default is sync). + """ + return self._transport + + @property + def options(self): + """ Options to be used to configure the exporter + """ + return self._options + + def export(self, span_datas): + """ + :type span_datas: list of :class: + `~opencensus.trace.span_data.SpanData` + :param list of opencensus.trace.span_data.SpanData span_datas: + SpanData tuples to export + """ + if span_datas is not None: # pragma: NO COVER + self.transport.export(span_datas) + + def emit(self, span_datas): + """ + :type span_datas: list of :class: + `~opencensus.trace.span_data.SpanData` + :param list of opencensus.trace.span_data.SpanData span_datas: + SpanData tuples to emit + """ + # Map each span data to it's corresponding trace id + trace_span_map = defaultdict(list) + for sd in span_datas: + trace_span_map[sd.context.trace_id] += [sd] + + dd_spans = [] + # Write spans to Datadog + for _, sds in trace_span_map.items(): + # convert to the legacy trace json for easier refactoring + trace = span_data.format_legacy_trace_json(sds) + dd_spans.append(self.translate_to_datadog(trace)) + + self._dd_transport.send_traces(dd_spans) + + def translate_to_datadog(self, trace): + """Translate the spans json to Datadog format. + + :type trace: dict + :param trace: Trace dictionary + + :rtype: dict + :returns: Spans in Datadog Trace format. + """ + + spans_json = trace.get('spans') + trace_id = convert_id(trace.get('traceId')[8:]) + dd_trace = [] + for span in spans_json: + span_id_int = convert_id(span.get('spanId')) + # Set meta at the end. + meta = self.options.global_tags.copy() + + dd_span = { + 'span_id': span_id_int, + 'trace_id': trace_id, + 'name': "opencensus", + 'service': self.options.service, + 'resource': span.get("displayName").get("value"), + } + + start_time = datetime.strptime(span.get('startTime'), + ISO_DATETIME_REGEX) + + # The start time of the request in nanoseconds from the unix epoch. + epoch = datetime.utcfromtimestamp(0) + dd_span["start"] = int((start_time - epoch).total_seconds() * + 1000.0 * 1000.0 * 1000.0) + + end_time = datetime.strptime(span.get('endTime'), + ISO_DATETIME_REGEX) + duration_td = end_time - start_time + + # The duration of the request in nanoseconds. + dd_span["duration"] = int(duration_td.total_seconds() * 1000.0 * + 1000.0 * 1000.0) + + if span.get('parentSpanId') is not None: + parent_span_id = convert_id(span.get('parentSpanId')) + dd_span['parent_id'] = parent_span_id + + code = STATUS_CODES.get(span["status"].get("code")) + if code is None: + code = {} + code["message"] = "ERR_CODE_" + str(span["status"].get("code")) + code["status"] = 500 + + # opencensus.trace.span.SpanKind + dd_span['type'] = to_dd_type(span.get("kind")) + dd_span["error"] = 0 + if 4 <= code.get("status") // 100 <= 5: + dd_span["error"] = 1 + meta["error.type"] = code.get("message") + + if span.get("status").get("message") is not None: + meta["error.msg"] = span.get("status").get("message") + + meta["opencensus.status_code"] = str(code.get("status")) + meta["opencensus.status"] = code.get("message") + + if span.get("status").get("message") is not None: + meta["opencensus.status_description"] = span.get("status").get( + "message") + + atts = span.get("attributes").get("attributeMap") + atts_to_metadata(atts, meta=meta) + + dd_span["meta"] = meta + dd_trace.append(dd_span) + + return dd_trace + + +def atts_to_metadata(atts, meta={}): + """Translate the attributes to Datadog meta format. + + :type atts: dict + :param atts: Attributes dictionary + + :rtype: dict + :returns: meta dictionary + """ + for key, elem in atts.items(): + value = value_from_atts_elem(elem) + if value != "": + meta[key] = value + + return meta + + +def value_from_atts_elem(elem): + """ value_from_atts_elem takes an attribute element and retuns a string value + + :type elem: dict + :param elem: Element from the attributes map + + :rtype: str + :return: A string rep of the element value + """ + if elem.get('string_value') is not None: + return elem.get('string_value').get('value') + elif elem.get('int_value') is not None: + return str(elem.get('int_value')) + elif elem.get('bool_value') is not None: + return str(elem.get('bool_value')) + elif elem.get('double_value') is not None: + return str(elem.get('double_value').get('value')) + return "" + + +def to_dd_type(oc_kind): + """ to_dd_type takes an OC kind int ID and returns a dd string of the span type + + :type oc_kind: int + :param oc_kind: OC kind id + + :rtype: string + :returns: A string of the Span type. + """ + if oc_kind == 2: + return "client" + elif oc_kind == 1: + return "server" + else: + return "unspecified" + + +def new_trace_exporter(option): + """ new_trace_exporter returns an exporter + that exports traces to Datadog. + """ + if option.service == "": + raise ValueError("Service can not be empty string.") + + exporter = DatadogTraceExporter(options=option) + return exporter + + +def convert_id(str_id): + """ convert_id takes a string and converts that to an int that is no + more than 64 bits wide. It does this by first converting the string + to a bit array then taking up to the 64th bit and creating and int. + This is equivlent to the go-exporter ID converter + ` binary.BigEndian.Uint64(s.SpanContext.SpanID[:])` + + :type str_id: str + :param str_id: string id + + :rtype: int + :returns: An int that is no more than 64 bits wide + """ + id_bitarray = bitarray.bitarray(endian='big') + id_bitarray.frombytes(str_id.encode()) + cut_off = len(id_bitarray) + if cut_off > 64: + cut_off = 64 + id_cutoff_bytearray = id_bitarray[:cut_off].tobytes() + id_int = int(codecs.encode(id_cutoff_bytearray, 'hex'), 16) + return id_int + + +# https://opencensus.io/tracing/span/status/ +STATUS_CODES = { + 0: { + "message": "OK", + "status": 200 + }, + 1: { + "message": "CANCELLED", + "status": 499 + }, + 2: { + "message": "UNKNOWN", + "status": 500 + }, + 3: { + "message": "INVALID_ARGUMENT", + "status": 400 + }, + 4: { + "message": "DEADLINE_EXCEEDED", + "status": 504 + }, + 5: { + "message": "NOT_FOUND", + "status": 404 + }, + 6: { + "message": "ALREADY_EXISTS", + "status": 409 + }, + 7: { + "message": "PERMISSION_DENIED", + "status": 403 + }, + 8: { + "message": "RESOURCE_EXHAUSTED", + "status": 429 + }, + 9: { + "message": "FAILED_PRECONDITION", + "status": 400 + }, + 10: { + "message": "ABORTED", + "status": 409 + }, + 11: { + "message": "OUT_OF_RANGE", + "status": 400 + }, + 12: { + "message": "UNIMPLEMENTED", + "status": 502 + }, + 13: { + "message": "INTERNAL", + "status": 500 + }, + 14: { + "message": "UNAVAILABLE", + "status": 503 + }, + 15: { + "message": "DATA_LOSS", + "status": 501 + }, + 16: { + "message": "UNAUTHENTICATED", + "status": 401 + }, +} diff --git a/contrib/opencensus-ext-datadog/opencensus/ext/datadog/transport.py b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/transport.py new file mode 100644 index 000000000..58c23fc6a --- /dev/null +++ b/contrib/opencensus-ext-datadog/opencensus/ext/datadog/transport.py @@ -0,0 +1,46 @@ +import platform + +import requests + + +class DDTransport(object): + """ DDTransport contains all the logic for sending Traces to Datadog + + :type trace_addr: str + :param trace_addr: trace_addr specifies the host[:port] address of the + Datadog Trace Agent. + """ + def __init__(self, trace_addr): + self._trace_addr = trace_addr + + self._headers = { + "Datadog-Meta-Lang": "python", + "Datadog-Meta-Lang-Interpreter": platform.platform(), + # Following the example of the Golang version it is prefixed + # OC for Opencensus. + "Datadog-Meta-Tracer-Version": "OC/0.0.1", + "Content-Type": "application/json", + } + + @property + def trace_addr(self): + """ specifies the host[:port] address of the Datadog Trace Agent. + """ + return self._trace_addr + + @property + def headers(self): + """ specifies the headers that will be attached to HTTP request sent to DD. + """ + return self._headers + + def send_traces(self, trace): + """ Sends traces to the Datadog Tracing Agent + + :type trace: dic + :param trace: Trace dictionary + """ + + requests.post("http://" + self.trace_addr + "/v0.4/traces", + json=trace, + headers=self.headers) diff --git a/contrib/opencensus-ext-datadog/setup.cfg b/contrib/opencensus-ext-datadog/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/contrib/opencensus-ext-datadog/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/contrib/opencensus-ext-datadog/setup.py b/contrib/opencensus-ext-datadog/setup.py new file mode 100644 index 000000000..790f5500e --- /dev/null +++ b/contrib/opencensus-ext-datadog/setup.py @@ -0,0 +1,56 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import find_packages, setup + +from version import __version__ + +setup( + name='opencensus-ext-datadog', + version=__version__, # noqa + author='OpenCensus Authors', + author_email='census-developers@googlegroups.com', + classifiers=[ + 'Intended Audience :: Developers', + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + ], + description='OpenCensus Datadog exporter', + include_package_data=True, + install_requires=[ + 'bitarray >= 1.0.1, < 2.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', + 'requests >= 2.19.0', + ], + extras_require={}, + license='Apache-2.0', + packages=find_packages(exclude=( + 'examples', + 'tests', + )), + namespace_packages=[], + url='https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-datadog', # noqa: E501 + zip_safe=False, +) diff --git a/contrib/opencensus-ext-datadog/tests/traces_test.py b/contrib/opencensus-ext-datadog/tests/traces_test.py new file mode 100644 index 000000000..4183c144d --- /dev/null +++ b/contrib/opencensus-ext-datadog/tests/traces_test.py @@ -0,0 +1,357 @@ +# import unittest + +# import mock + +# from opencensus.ext.datadog.traces import ( +# DatadogTraceExporter, +# Options, +# atts_to_metadata, +# convert_id, +# new_trace_exporter, +# to_dd_type, +# value_from_atts_elem, +# ) +# from opencensus.trace import span_context +# from opencensus.trace import span_data as span_data_module + + +# class TestTraces(unittest.TestCase): +# def setUp(self): +# pass + +# def test_convert_id(self): +# test_cases = [{ +# 'input': 'd17b83f89a2cbb08c2fa4469', +# 'expected': 0x6431376238336638, +# }, { +# 'input': '1ff346aeb5d12443', +# 'expected': 0x3166663334366165, +# }, { +# 'input': '8c9b71d2ffb05ede97bea00a', +# 'expected': 0x3863396237316432, +# }, { +# 'input': 'a3e1b9b4ce7d2e33', +# 'expected': 0x6133653162396234, +# }, { +# 'input': '2f79a1a078c0a4d070094440', +# 'expected': 0x3266373961316130, +# }, { +# 'input': '0018b3f50e44f875', +# 'expected': 0x3030313862336635, +# }, { +# 'input': 'cba7b2832de221dbc1ac8e77', +# 'expected': 0x6362613762323833, +# }, { +# 'input': 'a3e1b9b4', +# 'expected': 0x6133653162396234, +# }] +# for tc in test_cases: +# self.assertEqual(convert_id(tc['input']), tc['expected']) + +# def test_to_dd_type(self): +# self.assertEqual(to_dd_type(1), "server") +# self.assertEqual(to_dd_type(2), "client") +# self.assertEqual(to_dd_type(3), "unspecified") + +# def test_value_from_atts_elem(self): +# test_cases = [{ +# 'elem': { +# 'string_value': { +# 'value': 'StringValue' +# } +# }, +# 'expected': 'StringValue' +# }, { +# 'elem': { +# 'int_value': 10 +# }, +# 'expected': '10' +# }, { +# 'elem': { +# 'bool_value': True +# }, +# 'expected': 'True' +# }, { +# 'elem': { +# 'bool_value': False +# }, +# 'expected': 'False' +# }, { +# 'elem': { +# 'double_value': { +# 'value': 2.1 +# } +# }, +# 'expected': '2.1' +# }, { +# 'elem': { +# 'somthing_les': 2.1 +# }, +# 'expected': '' +# }] + +# for tc in test_cases: +# self.assertEqual(value_from_atts_elem(tc['elem']), tc['expected']) # noqa: E501 + +# def test_export(self): +# mock_dd_transport = mock.Mock() +# exporter = DatadogTraceExporter(options=Options(), +# transport=MockTransport) +# exporter._dd_transport = mock_dd_transport +# exporter.export({}) +# self.assertTrue(exporter.transport.export_called) + +# @mock.patch('opencensus.ext.datadog.traces.' +# 'DatadogTraceExporter.translate_to_datadog', +# return_value=None) +# def test_emit(self, mr_mock): + +# trace_id = '6e0c63257de34c92bf9efcd03927272e' +# span_datas = [ +# span_data_module.SpanData( +# name='span', +# context=span_context.SpanContext(trace_id=trace_id), +# span_id=None, +# parent_span_id=None, +# attributes=None, +# start_time=None, +# end_time=None, +# child_span_count=None, +# stack_trace=None, +# annotations=None, +# message_events=None, +# links=None, +# status=None, +# same_process_as_parent_span=None, +# span_kind=0, +# ) +# ] + +# mock_dd_transport = mock.Mock() +# exporter = DatadogTraceExporter( +# options=Options(service="dd-unit-test"), +# transport=MockTransport) +# exporter._dd_transport = mock_dd_transport + +# exporter.emit(span_datas) +# # mock_dd_transport.send_traces.assert_called_with(datadog_spans) +# self.assertTrue(mock_dd_transport.send_traces.called) + +# def test_translate_to_datadog(self): +# test_cases = [ +# { +# 'status': {'code': 0}, +# 'prt_span_id': '6e0c63257de34c92', +# 'expt_prt_span_id': 0x3665306336333235, +# 'attributes': { +# 'attributeMap': { +# 'key': { +# 'string_value': { +# 'truncated_byte_count': 0, +# 'value': 'value' +# } +# }, +# 'key_double': { +# 'double_value': { +# 'value': 123.45 +# } +# }, +# 'http.host': { +# 'string_value': { +# 'truncated_byte_count': 0, +# 'value': 'host' +# } +# } +# } +# }, +# 'meta': { +# 'key': 'value', +# 'key_double': '123.45', +# 'http.host': 'host', +# 'opencensus.status': 'OK', +# 'opencensus.status_code': '200' +# }, +# 'error': 0 +# }, +# { +# 'status': {'code': 23}, +# 'attributes': { +# 'attributeMap': {} +# }, +# 'meta': { +# 'error.type': 'ERR_CODE_23', +# 'opencensus.status': 'ERR_CODE_23', +# 'opencensus.status_code': '500' +# }, +# 'error': 1 +# }, +# { +# 'status': {'code': 23, 'message': 'I_AM_A_TEAPOT'}, +# 'attributes': { +# 'attributeMap': {} +# }, +# 'meta': { +# 'error.type': 'ERR_CODE_23', +# 'opencensus.status': 'ERR_CODE_23', +# 'opencensus.status_code': '500', +# 'opencensus.status_description': 'I_AM_A_TEAPOT', +# 'error.msg': 'I_AM_A_TEAPOT' +# }, +# 'error': 1 +# }, +# { +# 'status': {'code': 0, 'message': 'OK'}, +# 'attributes': { +# 'attributeMap': {} +# }, +# 'meta': { +# 'opencensus.status': 'OK', +# 'opencensus.status_code': '200', +# 'opencensus.status_description': 'OK' +# }, +# 'error': 0 +# } +# ] +# trace_id = '6e0c63257de34c92bf9efcd03927272e' +# expected_trace_id = 0x3764653334633932 +# span_id = '6e0c63257de34c92' +# expected_span_id = 0x3665306336333235 +# span_name = 'test span' +# start_time = '2019-09-19T14:05:15.000000Z' +# start_time_epoch = 1568901915000000000 +# end_time = '2019-09-19T14:05:16.000000Z' +# span_duration = 1 * 1000 * 1000 * 1000 + +# for tc in test_cases: +# mock_dd_transport = mock.Mock() +# opts = Options(service="dd-unit-test") +# tran = MockTransport +# exporter = DatadogTraceExporter(options=opts, transport=tran) +# exporter._dd_transport = mock_dd_transport +# trace = { +# 'spans': [{ +# 'displayName': { +# 'value': span_name, +# 'truncated_byte_count': 0 +# }, +# 'spanId': span_id, +# 'startTime': start_time, +# 'endTime': end_time, +# 'parentSpanId': tc.get('prt_span_id'), +# 'attributes': tc.get('attributes'), +# 'someRandomKey': 'this should not be included in result', +# 'childSpanCount': 0, +# 'kind': 1, +# 'status': tc.get('status') +# }], +# 'traceId': +# trace_id, +# } + +# spans = list(exporter.translate_to_datadog(trace)) +# expected_traces = [{ +# 'span_id': expected_span_id, +# 'trace_id': expected_trace_id, +# 'name': 'opencensus', +# 'service': 'dd-unit-test', +# 'resource': span_name, +# 'start': start_time_epoch, +# 'duration': span_duration, +# 'meta': tc.get('meta'), +# 'type': 'server', +# 'error': tc.get('error') +# }] + +# if tc.get('prt_span_id') is not None: +# expected_traces[0]['parent_id'] = tc.get('expt_prt_span_id') +# self.assertEqual.__self__.maxDiff = None +# self.assertEqual(spans, expected_traces) + +# def test_atts_to_metadata(self): +# test_cases = [ +# { +# 'input': { +# 'key_string': { +# 'string_value': { +# 'truncated_byte_count': 0, +# 'value': 'value' +# } +# }, +# 'key_double': { +# 'double_value': { +# 'value': 123.45 +# } +# }, +# }, +# 'input_meta': {}, +# 'output': { +# 'key_string': 'value', +# 'key_double': '123.45' +# } +# }, +# { +# 'input': { +# 'key_string': { +# 'string_value': { +# 'truncated_byte_count': 0, +# 'value': 'value' +# } +# }, +# }, +# 'input_meta': { +# 'key': 'in_meta' +# }, +# 'output': { +# 'key_string': 'value', +# 'key': 'in_meta' +# } +# }, +# { +# 'input': { +# 'key_string': { +# 'string_value': { +# 'truncated_byte_count': 0, +# 'value': 'value' +# } +# }, +# 'invalid': { +# 'unknown_value': "na" +# } +# }, +# 'input_meta': {}, +# 'output': { +# 'key_string': 'value', +# } +# } +# ] + +# for tc in test_cases: +# out = atts_to_metadata(tc.get('input'), meta=tc.get('input_meta')) # noqa: E501 +# self.assertEqual(out, tc.get('output')) + +# def test_new_trace_exporter(self): +# self.assertRaises(ValueError, new_trace_exporter, Options()) +# try: +# new_trace_exporter(Options(service="test")) +# except ValueError: +# self.fail("new_trace_exporter raised ValueError unexpectedly") + +# def test_constructure(self): +# self.assertRaises(TypeError, Options, global_tags={'int_bad': 1}) +# try: +# Options(global_tags={'good': 'tag'}) +# except TypeError: +# self.fail("Constructure raised TypeError unexpectedly") + + +# class MockTransport(object): +# def __init__(self, exporter=None): +# self.export_called = False +# self.exporter = exporter + +# def export(self, trace): +# self.export_called = True + + +# if __name__ == '__main__': +# unittest.main() diff --git a/contrib/opencensus-ext-datadog/tests/transport_test.py b/contrib/opencensus-ext-datadog/tests/transport_test.py new file mode 100644 index 000000000..c71ded03a --- /dev/null +++ b/contrib/opencensus-ext-datadog/tests/transport_test.py @@ -0,0 +1,15 @@ +# import unittest + +# import mock + +# from opencensus.ext.datadog.transport import DDTransport + + +# class TestTraces(unittest.TestCase): +# def setUp(self): +# pass + +# @mock.patch('requests.post', return_value=None) +# def test_send_traces(self, mr_mock): +# transport = DDTransport('test') +# transport.send_traces({}) diff --git a/contrib/opencensus-ext-requests/version.py b/contrib/opencensus-ext-datadog/version.py similarity index 100% rename from contrib/opencensus-ext-requests/version.py rename to contrib/opencensus-ext-datadog/version.py diff --git a/contrib/opencensus-ext-dbapi/setup.py b/contrib/opencensus-ext-dbapi/setup.py index 1583edbf6..a1ac8b0ee 100644 --- a/contrib/opencensus-ext-dbapi/setup.py +++ b/contrib/opencensus-ext-dbapi/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Database API Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-django/CHANGELOG.md b/contrib/opencensus-ext-django/CHANGELOG.md index cfc2dddb1..54ff9668e 100644 --- a/contrib/opencensus-ext-django/CHANGELOG.md +++ b/contrib/opencensus-ext-django/CHANGELOG.md @@ -1,6 +1,61 @@ # Changelog ## Unreleased + +## 0.8.0 +Released 2022-10-17 + +- Fixed support for Django 4.1 +- ([#1159](https://github.com/census-instrumentation/opencensus-python/pull/1159)) + +## 0.7.5 +Released 2021-05-13 + +- Add exception tracing to django middleware +([#885](https://github.com/census-instrumentation/opencensus-python/pull/885)) + +## 0.7.4 +Released 2021-01-19 + +- Hotfix +([#999](https://github.com/census-instrumentation/opencensus-python/pull/999)) + +## 0.7.3 +Released 2021-01-14 + +- Change blacklist to excludelist +([#977](https://github.com/census-instrumentation/opencensus-python/pull/977)) + +## 0.7.2 +Released 2019-09-30 + +- Use Django 2.0 DB instrumentation +([#775](https://github.com/census-instrumentation/opencensus-python/pull/775)) + +## 0.7.1 +Released 2019-08-26 + +- Updated `http.status_code` attribute to be an int. + ([#755](https://github.com/census-instrumentation/opencensus-python/pull/755)) + +## 0.7.0 +Released 2019-07-31 + +- Updated span attributes to include some missing attributes listed + [here](https://github.com/census-instrumentation/opencensus-specs/blob/master/trace/HTTP.md#attributes) + ([#735](https://github.com/census-instrumentation/opencensus-python/pull/735)) + +## 0.3.2 +Released 2019-07-26 + +- Removed support for Django < 1.11 + ([#694](https://github.com/census-instrumentation/opencensus-python/pull/694)) +- Allow installing with Django 2.0 and later + ([#697](https://github.com/census-instrumentation/opencensus-python/pull/697)) + +## 0.3.1 +Released 2019-06-04 + - Make ProbabilitySampler default ## 0.3.0 diff --git a/contrib/opencensus-ext-django/README.rst b/contrib/opencensus-ext-django/README.rst index 9fe978fc6..13601efb1 100644 --- a/contrib/opencensus-ext-django/README.rst +++ b/contrib/opencensus-ext-django/README.rst @@ -17,24 +17,15 @@ Usage ----- For tracing Django requests, you will need to add the following line to -the ``MIDDLEWARE_CLASSES`` section in the Django ``settings.py`` file. +the ``MIDDLEWARE`` section in the Django ``settings.py`` file. .. code:: python - MIDDLEWARE_CLASSES = [ + MIDDLEWARE = [ ... 'opencensus.ext.django.middleware.OpencensusMiddleware', ] -And add this line to the ``INSTALLED_APPS`` section: - -.. code:: python - - INSTALLED_APPS = [ - ... - 'opencensus.ext.django', - ] - Additional configuration can be provided, please read `Customization `_ for a complete reference. diff --git a/contrib/opencensus-ext-django/examples/app/settings.py b/contrib/opencensus-ext-django/examples/app/settings.py index ec039902b..3a391eb8a 100644 --- a/contrib/opencensus-ext-django/examples/app/settings.py +++ b/contrib/opencensus-ext-django/examples/app/settings.py @@ -29,19 +29,17 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'opencensus.trace.ext.django', ) -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.security.SecurityMiddleware', - 'opencensus.trace.ext.django.middleware.OpencensusMiddleware', + 'opencensus.ext.django.middleware.OpencensusMiddleware', ) ROOT_URLCONF = 'app.urls' diff --git a/contrib/opencensus-ext-django/examples/app/urls.py b/contrib/opencensus-ext-django/examples/app/urls.py index 7d3f35b22..bb46753f1 100644 --- a/contrib/opencensus-ext-django/examples/app/urls.py +++ b/contrib/opencensus-ext-django/examples/app/urls.py @@ -27,14 +27,13 @@ 1. Add an import: from blog import urls as blog_urls 2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls)) """ -from django.conf.urls import include, url +from django.conf.urls import url from django.contrib import admin import app.views - urlpatterns = [ - url(r'^admin/', include(admin.site.urls)), + url(r'^admin/', admin.site.urls), url(r'^$', app.views.home), url(r'^greetings$', app.views.greetings), url(r'^_ah/health$', app.views.health_check), diff --git a/contrib/opencensus-ext-django/examples/app/views.py b/contrib/opencensus-ext-django/examples/app/views.py index ba01e98c1..a940a7932 100644 --- a/contrib/opencensus-ext-django/examples/app/views.py +++ b/contrib/opencensus-ext-django/examples/app/views.py @@ -12,19 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from django.http import HttpResponse -from django.shortcuts import render - -from .forms import HelloForm - -from opencensus.trace import config_integration +import os import mysql.connector import psycopg2 +import requests import sqlalchemy +from django.http import HttpResponse +from django.shortcuts import render -import os -import requests +from opencensus.trace import config_integration + +from .forms import HelloForm DB_HOST = 'localhost' @@ -59,7 +58,7 @@ def greetings(request): def trace_requests(request): response = requests.get('http://www.google.com') - return HttpResponse(str(response.status_code)) + return HttpResponse(response.status_code) def mysql_trace(request): diff --git a/contrib/opencensus-ext-django/opencensus/ext/django/middleware.py b/contrib/opencensus-ext-django/opencensus/ext/django/middleware.py index 886e6b04f..e9c116983 100644 --- a/contrib/opencensus-ext-django/opencensus/ext/django/middleware.py +++ b/contrib/opencensus-ext-django/opencensus/ext/django/middleware.py @@ -13,35 +13,47 @@ # limitations under the License. """Django middleware helper to capture and trace a request.""" -import logging import six +import logging +import sys +import traceback + +import django import django.conf +from django.db import connection +from django.utils.deprecation import MiddlewareMixin +from google.rpc import code_pb2 from opencensus.common import configuration -from opencensus.trace import attributes_helper -from opencensus.trace import execution_context -from opencensus.trace import print_exporter -from opencensus.trace import samplers +from opencensus.trace import ( + attributes_helper, + execution_context, + integrations, + print_exporter, + samplers, +) from opencensus.trace import span as span_module +from opencensus.trace import status as status_module from opencensus.trace import tracer as tracer_module from opencensus.trace import utils from opencensus.trace.propagation import trace_context_http_header_format -try: - from django.utils.deprecation import MiddlewareMixin -except ImportError: # pragma: NO COVER - MiddlewareMixin = object - +HTTP_HOST = attributes_helper.COMMON_ATTRIBUTES['HTTP_HOST'] HTTP_METHOD = attributes_helper.COMMON_ATTRIBUTES['HTTP_METHOD'] +HTTP_PATH = attributes_helper.COMMON_ATTRIBUTES['HTTP_PATH'] +HTTP_ROUTE = attributes_helper.COMMON_ATTRIBUTES['HTTP_ROUTE'] HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL'] HTTP_STATUS_CODE = attributes_helper.COMMON_ATTRIBUTES['HTTP_STATUS_CODE'] +ERROR_MESSAGE = attributes_helper.COMMON_ATTRIBUTES['ERROR_MESSAGE'] +ERROR_NAME = attributes_helper.COMMON_ATTRIBUTES['ERROR_NAME'] +STACKTRACE = attributes_helper.COMMON_ATTRIBUTES['STACKTRACE'] REQUEST_THREAD_LOCAL_KEY = 'django_request' SPAN_THREAD_LOCAL_KEY = 'django_span' -BLACKLIST_PATHS = 'BLACKLIST_PATHS' -BLACKLIST_HOSTNAMES = 'BLACKLIST_HOSTNAMES' +EXCLUDELIST_PATHS = 'EXCLUDELIST_PATHS' +EXCLUDELIST_HOSTNAMES = 'EXCLUDELIST_HOSTNAMES' log = logging.getLogger(__name__) @@ -90,12 +102,7 @@ def _set_django_attributes(span, request): return user_id = django_user.pk - try: - user_name = django_user.get_username() - except AttributeError: - # AnonymousUser in some older versions of Django doesn't implement - # get_username - return + user_name = django_user.get_username() # User id is the django autofield for User model as the primary key if user_id is not None: @@ -105,11 +112,42 @@ def _set_django_attributes(span, request): span.add_attribute('django.user.name', str(user_name)) +def _trace_db_call(execute, sql, params, many, context): + tracer = _get_current_tracer() + if not tracer: + return execute(sql, params, many, context) + + vendor = context['connection'].vendor + alias = context['connection'].alias + + span = tracer.start_span() + span.name = '{}.query'.format(vendor) + span.span_kind = span_module.SpanKind.CLIENT + + tracer.add_attribute_to_current_span('component', vendor) + tracer.add_attribute_to_current_span('db.instance', alias) + tracer.add_attribute_to_current_span('db.statement', sql) + tracer.add_attribute_to_current_span('db.type', 'sql') + + try: + result = execute(sql, params, many, context) + except Exception: # pragma: NO COVER + status = status_module.Status( + code=code_pb2.UNKNOWN, message='DB error' + ) + span.set_status(status) + raise + else: + return result + finally: + tracer.end_span() + + class OpencensusMiddleware(MiddlewareMixin): """Saves the request in thread local""" - def __init__(self, get_response=None): - self.get_response = get_response + def __init__(self, get_response): + super(OpencensusMiddleware, self).__init__(get_response) settings = getattr(django.conf.settings, 'OPENCENSUS', {}) settings = settings.get('TRACE', {}) @@ -128,9 +166,15 @@ def __init__(self, get_response=None): if isinstance(self.propagator, six.string_types): self.propagator = configuration.load(self.propagator) - self.blacklist_paths = settings.get(BLACKLIST_PATHS, None) + self.excludelist_paths = settings.get(EXCLUDELIST_PATHS, None) - self.blacklist_hostnames = settings.get(BLACKLIST_HOSTNAMES, None) + self.excludelist_hostnames = settings.get(EXCLUDELIST_HOSTNAMES, None) + + if django.VERSION >= (2,): # pragma: NO COVER + connection.execute_wrappers.append(_trace_db_call) + + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.DJANGO) def process_request(self, request): """Called on each request, before Django decides which view to execute. @@ -138,8 +182,8 @@ def process_request(self, request): :type request: :class:`~django.http.request.HttpRequest` :param request: Django http request. """ - # Do not trace if the url is blacklisted - if utils.disable_tracing_url(request.path, self.blacklist_paths): + # Do not trace if the url is excludelisted + if utils.disable_tracing_url(request.path, self.excludelist_paths): return # Add the request to thread local @@ -148,8 +192,8 @@ def process_request(self, request): request) execution_context.set_opencensus_attr( - 'blacklist_hostnames', - self.blacklist_hostnames) + 'excludelist_hostnames', + self.excludelist_hostnames) try: # Start tracing this request @@ -166,12 +210,21 @@ def process_request(self, request): # Span name is being set at process_view span = tracer.start_span() span.span_kind = span_module.SpanKind.SERVER + tracer.add_attribute_to_current_span( + attribute_key=HTTP_HOST, + attribute_value=request.get_host()) tracer.add_attribute_to_current_span( attribute_key=HTTP_METHOD, attribute_value=request.method) tracer.add_attribute_to_current_span( - attribute_key=HTTP_URL, + attribute_key=HTTP_PATH, attribute_value=str(request.path)) + tracer.add_attribute_to_current_span( + attribute_key=HTTP_ROUTE, + attribute_value=str(request.path)) + tracer.add_attribute_to_current_span( + attribute_key=HTTP_URL, + attribute_value=str(request.build_absolute_uri())) # Add the span to thread local # in some cases (exceptions, timeouts) currentspan in @@ -190,8 +243,8 @@ def process_view(self, request, view_func, *args, **kwargs): function name add set it as the span name. """ - # Do not trace if the url is blacklisted - if utils.disable_tracing_url(request.path, self.blacklist_paths): + # Do not trace if the url is excludelisted + if utils.disable_tracing_url(request.path, self.excludelist_paths): return try: @@ -204,15 +257,15 @@ def process_view(self, request, view_func, *args, **kwargs): log.error('Failed to trace request', exc_info=True) def process_response(self, request, response): - # Do not trace if the url is blacklisted - if utils.disable_tracing_url(request.path, self.blacklist_paths): + # Do not trace if the url is excludelisted + if utils.disable_tracing_url(request.path, self.excludelist_paths): return response try: span = _get_django_span() span.add_attribute( attribute_key=HTTP_STATUS_CODE, - attribute_value=str(response.status_code)) + attribute_value=response.status_code) _set_django_attributes(span, request) @@ -223,3 +276,29 @@ def process_response(self, request, response): log.error('Failed to trace request', exc_info=True) finally: return response + + def process_exception(self, request, exception): + # Do not trace if the url is excluded + if utils.disable_tracing_url(request.path, self.excludelist_paths): + return + + try: + if hasattr(exception, '__traceback__'): + tb = exception.__traceback__ + else: + _, _, tb = sys.exc_info() + + span = _get_django_span() + span.add_attribute( + attribute_key=ERROR_NAME, + attribute_value=exception.__class__.__name__) + span.add_attribute( + attribute_key=ERROR_MESSAGE, + attribute_value=str(exception)) + span.add_attribute( + attribute_key=STACKTRACE, + attribute_value='\n'.join(traceback.format_tb(tb))) + + _set_django_attributes(span, request) + except Exception: # pragma: NO COVER + log.error('Failed to trace request', exc_info=True) diff --git a/contrib/opencensus-ext-django/setup.py b/contrib/opencensus-ext-django/setup.py index b89f3319c..9aa301e8b 100644 --- a/contrib/opencensus-ext-django/setup.py +++ b/contrib/opencensus-ext-django/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -26,6 +26,10 @@ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', + 'Framework :: Django', + 'Framework :: Django :: 1.11', + 'Framework :: Django :: 2.1', + 'Framework :: Django :: 2.2', 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', @@ -34,13 +38,15 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Django Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'Django >= 1.11.0, < 1.12.0', - 'opencensus >= 0.7.dev0, < 1.0.0', + 'Django >= 1.11', + 'opencensus >= 0.9.dev0, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-django/tests/test_django_db_middleware.py b/contrib/opencensus-ext-django/tests/test_django_db_middleware.py new file mode 100644 index 000000000..4c6969ae4 --- /dev/null +++ b/contrib/opencensus-ext-django/tests/test_django_db_middleware.py @@ -0,0 +1,94 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from collections import namedtuple + +import django +import mock +import pytest +from django.http import HttpResponse +from django.test.utils import teardown_test_environment + +from opencensus.trace import execution_context + + +def get_response(request): + return HttpResponse() + + +class TestOpencensusDatabaseMiddleware(unittest.TestCase): + def setUp(self): + from django.conf import settings as django_settings + from django.test.utils import setup_test_environment + + if not django_settings.configured: + django_settings.configure() + setup_test_environment() + + def tearDown(self): + execution_context.clear() + teardown_test_environment() + + def test_process_request(self): + if django.VERSION < (2, 0): + pytest.skip("Wrong version of Django") + + from opencensus.ext.django import middleware + + sql = "SELECT * FROM users" + + MockConnection = namedtuple('Connection', ('vendor', 'alias')) + connection = MockConnection('mysql', 'default') + + mock_execute = mock.Mock() + mock_execute.return_value = "Mock result" + + middleware.OpencensusMiddleware(get_response) + + patch_no_tracer = mock.patch( + 'opencensus.ext.django.middleware._get_current_tracer', + return_value=None) + with patch_no_tracer: + result = middleware._trace_db_call( + mock_execute, sql, params=[], many=False, + context={'connection': connection}) + self.assertEqual(result, "Mock result") + + mock_tracer = mock.Mock() + mock_tracer.return_value = mock_tracer + patch = mock.patch( + 'opencensus.ext.django.middleware._get_current_tracer', + return_value=mock_tracer) + with patch: + result = middleware._trace_db_call( + mock_execute, sql, params=[], many=False, + context={'connection': connection}) + + (mock_sql, mock_params, mock_many, + mock_context) = mock_execute.call_args[0] + + self.assertEqual(mock_sql, sql) + self.assertEqual(mock_params, []) + self.assertEqual(mock_many, False) + self.assertEqual(mock_context, {'connection': connection}) + self.assertEqual(result, "Mock result") + + result = middleware._trace_db_call( + mock_execute, sql, params=[], many=True, + context={'connection': connection}) + + (mock_sql, mock_params, mock_many, + mock_context) = mock_execute.call_args[0] + self.assertEqual(mock_many, True) diff --git a/contrib/opencensus-ext-django/tests/test_django_middleware.py b/contrib/opencensus-ext-django/tests/test_django_middleware.py index 8b415d4a4..e502e0be0 100644 --- a/contrib/opencensus-ext-django/tests/test_django_middleware.py +++ b/contrib/opencensus-ext-django/tests/test_django_middleware.py @@ -12,21 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +import sys +import traceback import unittest +import mock +from django.http import HttpResponse from django.test import RequestFactory from django.test.utils import teardown_test_environment -from opencensus.trace import execution_context -from opencensus.trace import print_exporter -from opencensus.trace import samplers +from opencensus.trace import execution_context, print_exporter, samplers from opencensus.trace import span as span_module from opencensus.trace import utils from opencensus.trace.blank_span import BlankSpan from opencensus.trace.propagation import trace_context_http_header_format +def get_response(request): + return HttpResponse() + + class TestOpencensusMiddleware(unittest.TestCase): def setUp(self): @@ -44,7 +49,7 @@ def tearDown(self): def test_constructor_default(self): from opencensus.ext.django import middleware - middleware = middleware.OpencensusMiddleware() + middleware = middleware.OpencensusMiddleware(get_response) assert isinstance(middleware.sampler, samplers.ProbabilitySampler) assert isinstance(middleware.exporter, print_exporter.PrintExporter) @@ -69,7 +74,7 @@ def test_configuration(self): settings) with patch_settings: - middleware = middleware.OpencensusMiddleware() + middleware = middleware.OpencensusMiddleware(get_response) assert isinstance(middleware.sampler, samplers.AlwaysOnSampler) assert isinstance(middleware.exporter, print_exporter.PrintExporter) @@ -85,7 +90,7 @@ def test_process_request(self): span_id = '6e0c63257de34c92' django_trace_id = '00-{}-{}-00'.format(trace_id, span_id) - django_request = RequestFactory().get('/', **{ + django_request = RequestFactory().get('/wiki/Rabbit', **{ 'HTTP_TRACEPARENT': django_trace_id}) # Force the test request to be sampled @@ -100,7 +105,7 @@ def test_process_request(self): settings) with patch_settings: - middleware_obj = middleware.OpencensusMiddleware() + middleware_obj = middleware.OpencensusMiddleware(get_response) # test process_request middleware_obj.process_request(django_request) @@ -110,8 +115,11 @@ def test_process_request(self): span = tracer.current_span() expected_attributes = { - 'http.url': u'/', + 'http.host': u'testserver', 'http.method': 'GET', + 'http.path': u'/wiki/Rabbit', + 'http.route': u'/wiki/Rabbit', + 'http.url': u'http://testserver/wiki/Rabbit', } self.assertEqual(span.span_kind, span_module.SpanKind.SERVER) self.assertEqual(span.attributes, expected_attributes) @@ -126,17 +134,17 @@ def test_process_request(self): self.assertEqual(span.name, 'mock.mock.Mock') - def test_blacklist_path(self): + def test_excludelist_path(self): from opencensus.ext.django import middleware execution_context.clear() - blacklist_paths = ['test_blacklist_path'] + excludelist_paths = ['test_excludelist_path'] settings = type('Test', (object,), {}) settings.OPENCENSUS = { 'TRACE': { 'SAMPLER': 'opencensus.trace.samplers.AlwaysOnSampler()', # noqa - 'BLACKLIST_PATHS': blacklist_paths, + 'EXCLUDELIST_PATHS': excludelist_paths, 'EXPORTER': mock.Mock(), } } @@ -145,13 +153,13 @@ def test_blacklist_path(self): settings) with patch_settings: - middleware_obj = middleware.OpencensusMiddleware() + middleware_obj = middleware.OpencensusMiddleware(get_response) - django_request = RequestFactory().get('/test_blacklist_path') + django_request = RequestFactory().get('/test_excludelist_path') disabled = utils.disable_tracing_url(django_request.path, - blacklist_paths) + excludelist_paths) self.assertTrue(disabled) - self.assertEqual(middleware_obj.blacklist_paths, blacklist_paths) + self.assertEqual(middleware_obj.excludelist_paths, excludelist_paths) # test process_request middleware_obj.process_request(django_request) @@ -185,7 +193,7 @@ def test_process_response(self): span_id = '6e0c63257de34c92' django_trace_id = '00-{}-{}-00'.format(trace_id, span_id) - django_request = RequestFactory().get('/', **{ + django_request = RequestFactory().get('/wiki/Rabbit', **{ 'traceparent': django_trace_id, }) @@ -201,7 +209,7 @@ def test_process_response(self): settings) with patch_settings: - middleware_obj = middleware.OpencensusMiddleware() + middleware_obj = middleware.OpencensusMiddleware(get_response) middleware_obj.process_request(django_request) tracer = middleware._get_current_tracer() @@ -214,9 +222,12 @@ def test_process_response(self): django_response.status_code = 200 expected_attributes = { - 'http.url': u'/', + 'http.host': u'testserver', 'http.method': 'GET', - 'http.status_code': '200', + 'http.path': u'/wiki/Rabbit', + 'http.route': u'/wiki/Rabbit', + 'http.url': u'http://testserver/wiki/Rabbit', + 'http.status_code': 200, 'django.user.id': '123', 'django.user.name': 'test_name' } @@ -230,14 +241,14 @@ def test_process_response(self): self.assertEqual(span.attributes, expected_attributes) - def test_process_response_no_get_username(self): + def test_process_response_unfinished_child_span(self): from opencensus.ext.django import middleware trace_id = '2dd43a1d6b2549c6bc2a1a54c2fc0b05' span_id = '6e0c63257de34c92' django_trace_id = '00-{}-{}-00'.format(trace_id, span_id) - django_request = RequestFactory().get('/', **{ + django_request = RequestFactory().get('/wiki/Rabbit', **{ 'traceparent': django_trace_id, }) @@ -253,7 +264,7 @@ def test_process_response_no_get_username(self): settings) with patch_settings: - middleware_obj = middleware.OpencensusMiddleware() + middleware_obj = middleware.OpencensusMiddleware(get_response) middleware_obj.process_request(django_request) tracer = middleware._get_current_tracer() @@ -263,31 +274,38 @@ def test_process_response_no_get_username(self): tracer.exporter = exporter_mock django_response = mock.Mock() - django_response.status_code = 200 + django_response.status_code = 500 expected_attributes = { - 'http.url': u'/', + 'http.host': u'testserver', 'http.method': 'GET', - 'http.status_code': '200', + 'http.path': u'/wiki/Rabbit', + 'http.route': u'/wiki/Rabbit', + 'http.url': u'http://testserver/wiki/Rabbit', + 'http.status_code': 500, + 'django.user.id': '123', + 'django.user.name': 'test_name' } mock_user = mock.Mock() mock_user.pk = 123 - mock_user.get_username.side_effect = AttributeError + mock_user.get_username.return_value = 'test_name' django_request.user = mock_user + tracer.start_span() + self.assertNotEqual(span, tracer.current_span()) middleware_obj.process_response(django_request, django_response) self.assertEqual(span.attributes, expected_attributes) - def test_process_response_unfinished_child_span(self): + def test_process_exception(self): from opencensus.ext.django import middleware trace_id = '2dd43a1d6b2549c6bc2a1a54c2fc0b05' span_id = '6e0c63257de34c92' django_trace_id = '00-{}-{}-00'.format(trace_id, span_id) - django_request = RequestFactory().get('/', **{ + django_request = RequestFactory().get('/wiki/Rabbit', **{ 'traceparent': django_trace_id, }) @@ -303,7 +321,17 @@ def test_process_response_unfinished_child_span(self): settings) with patch_settings: - middleware_obj = middleware.OpencensusMiddleware() + middleware_obj = middleware.OpencensusMiddleware(get_response) + + tb = None + try: + raise RuntimeError("bork bork bork") + except Exception as exc: + test_exception = exc + if hasattr(exc, "__traceback__"): + tb = exc.__traceback__ + else: + _, _, tb = sys.exc_info() middleware_obj.process_request(django_request) tracer = middleware._get_current_tracer() @@ -313,14 +341,19 @@ def test_process_response_unfinished_child_span(self): tracer.exporter = exporter_mock django_response = mock.Mock() - django_response.status_code = 500 + django_response.status_code = 200 expected_attributes = { - 'http.url': u'/', + 'http.host': u'testserver', 'http.method': 'GET', - 'http.status_code': '500', + 'http.path': u'/wiki/Rabbit', + 'http.route': u'/wiki/Rabbit', + 'http.url': u'http://testserver/wiki/Rabbit', 'django.user.id': '123', - 'django.user.name': 'test_name' + 'django.user.name': 'test_name', + 'error.name': "RuntimeError", + 'error.message': 'bork bork bork', + 'stacktrace': '\n'.join(traceback.format_tb(tb)) } mock_user = mock.Mock() @@ -328,9 +361,7 @@ def test_process_response_unfinished_child_span(self): mock_user.get_username.return_value = 'test_name' django_request.user = mock_user - tracer.start_span() - self.assertNotEqual(span, tracer.current_span()) - middleware_obj.process_response(django_request, django_response) + middleware_obj.process_exception(django_request, test_exception) self.assertEqual(span.attributes, expected_attributes) diff --git a/contrib/opencensus-ext-django/version.py b/contrib/opencensus-ext-django/version.py index deb2f374d..671fc3d04 100644 --- a/contrib/opencensus-ext-django/version.py +++ b/contrib/opencensus-ext-django/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.4.dev0' +__version__ = '0.9.dev0' diff --git a/contrib/opencensus-ext-fastapi/CHANGELOG.md b/contrib/opencensus-ext-fastapi/CHANGELOG.md new file mode 100644 index 000000000..f4c2570de --- /dev/null +++ b/contrib/opencensus-ext-fastapi/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## Unreleased + +## 0.1.0 + +Released 2023-03-10 + +- Initial version +([#1124](https://github.com/census-instrumentation/opencensus-python/pull/1124)) diff --git a/contrib/opencensus-ext-fastapi/README.rst b/contrib/opencensus-ext-fastapi/README.rst new file mode 100644 index 000000000..7946d56ed --- /dev/null +++ b/contrib/opencensus-ext-fastapi/README.rst @@ -0,0 +1,50 @@ +OpenCensus FastAPI Integration +============================================================================ + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opencensus-ext-fastapi.svg + :target: https://pypi.org/project/opencensus-ext-fastapi/ + +Installation +------------ + +:: + + pip install opencensus-ext-fastapi + +Usage +----- + +.. code:: python + + from fastapi import FastAPI + from opencensus.ext.fastapi.fastapi_middleware import FastAPIMiddleware + + app = FastAPI(__name__) + app.add_middleware(FastAPIMiddleware) + + @app.get('/') + def hello(): + return 'Hello World!' + +Additional configuration can be provided, please read +`Customization `_ +for a complete reference. + +.. code:: python + + app.add_middleware( + FastAPIMiddleware, + excludelist_paths=["paths"], + excludelist_hostnames=["hostnames"], + sampler=sampler, + exporter=exporter, + propagator=propagator, + ) + + +References +---------- + +* `OpenCensus Project `_ diff --git a/contrib/opencensus-ext-fastapi/opencensus/__init__.py b/contrib/opencensus-ext-fastapi/opencensus/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-fastapi/opencensus/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-fastapi/opencensus/ext/__init__.py b/contrib/opencensus-ext-fastapi/opencensus/ext/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-fastapi/opencensus/ext/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-fastapi/opencensus/ext/fastapi/__init__.py b/contrib/opencensus-ext-fastapi/opencensus/ext/fastapi/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-fastapi/opencensus/ext/fastapi/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-fastapi/opencensus/ext/fastapi/fastapi_middleware.py b/contrib/opencensus-ext-fastapi/opencensus/ext/fastapi/fastapi_middleware.py new file mode 100644 index 000000000..6dfd1a812 --- /dev/null +++ b/contrib/opencensus-ext-fastapi/opencensus/ext/fastapi/fastapi_middleware.py @@ -0,0 +1,182 @@ +# Copyright 2022, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import traceback +from typing import Union + +from starlette.middleware.base import ( + BaseHTTPMiddleware, + RequestResponseEndpoint, +) +from starlette.requests import Request +from starlette.responses import Response +from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR +from starlette.types import ASGIApp + +from opencensus.trace import ( + attributes_helper, + execution_context, + integrations, + print_exporter, + samplers, +) +from opencensus.trace import span as span_module +from opencensus.trace import tracer as tracer_module +from opencensus.trace import utils +from opencensus.trace.blank_span import BlankSpan +from opencensus.trace.propagation import trace_context_http_header_format +from opencensus.trace.span import Span + +HTTP_HOST = attributes_helper.COMMON_ATTRIBUTES["HTTP_HOST"] +HTTP_METHOD = attributes_helper.COMMON_ATTRIBUTES["HTTP_METHOD"] +HTTP_PATH = attributes_helper.COMMON_ATTRIBUTES["HTTP_PATH"] +HTTP_ROUTE = attributes_helper.COMMON_ATTRIBUTES["HTTP_ROUTE"] +HTTP_URL = attributes_helper.COMMON_ATTRIBUTES["HTTP_URL"] +HTTP_STATUS_CODE = attributes_helper.COMMON_ATTRIBUTES["HTTP_STATUS_CODE"] +ERROR_MESSAGE = attributes_helper.COMMON_ATTRIBUTES['ERROR_MESSAGE'] +ERROR_NAME = attributes_helper.COMMON_ATTRIBUTES['ERROR_NAME'] +STACKTRACE = attributes_helper.COMMON_ATTRIBUTES["STACKTRACE"] + +module_logger = logging.getLogger(__name__) + + +class FastAPIMiddleware(BaseHTTPMiddleware): + """FastAPI middleware to automatically trace requests. + + :type app: :class: `~fastapi.FastAPI` + :param app: A fastapi application. + + :type excludelist_paths: list + :param excludelist_paths: Paths that do not trace. + + :type excludelist_hostnames: list + :param excludelist_hostnames: Hostnames that do not trace. + + :type sampler: :class:`~opencensus.trace.samplers.base.Sampler` + :param sampler: A sampler. It should extend from the base + :class:`.Sampler` type and implement + :meth:`.Sampler.should_sample`. Defaults to + :class:`.ProbabilitySampler`. Other options include + :class:`.AlwaysOnSampler` and :class:`.AlwaysOffSampler`. + + :type exporter: :class:`~opencensus.trace.base_exporter.exporter` + :param exporter: An exporter. Default to + :class:`.PrintExporter`. The rest options are + :class:`.FileExporter`, :class:`.LoggingExporter` and + trace exporter extensions. + + :type propagator: :class: 'object' + :param propagator: A propagator. Default to + :class:`.TraceContextPropagator`. The rest options + are :class:`.BinaryFormatPropagator`, + :class:`.GoogleCloudFormatPropagator` and + :class:`.TextFormatPropagator`. + """ + + def __init__( + self, + app: ASGIApp, + excludelist_paths=None, + excludelist_hostnames=None, + sampler=None, + exporter=None, + propagator=None, + ) -> None: + super().__init__(app) + self.excludelist_paths = excludelist_paths + self.excludelist_hostnames = excludelist_hostnames + self.sampler = sampler or samplers.AlwaysOnSampler() + self.exporter = exporter or print_exporter.PrintExporter() + self.propagator = ( + propagator or + trace_context_http_header_format.TraceContextPropagator() + ) + + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.FASTAPI) + + def _prepare_tracer(self, request: Request) -> tracer_module.Tracer: + span_context = self.propagator.from_headers(request.headers) + tracer = tracer_module.Tracer( + span_context=span_context, + sampler=self.sampler, + exporter=self.exporter, + propagator=self.propagator, + ) + return tracer + + def _before_request(self, span: Union[Span, BlankSpan], request: Request): + span.span_kind = span_module.SpanKind.SERVER + span.name = "[{}]{}".format(request.method, request.url) + span.add_attribute(HTTP_HOST, request.url.hostname) + span.add_attribute(HTTP_METHOD, request.method) + span.add_attribute(HTTP_PATH, request.url.path) + span.add_attribute(HTTP_URL, str(request.url)) + span.add_attribute(HTTP_ROUTE, request.url.path) + execution_context.set_opencensus_attr( + "excludelist_hostnames", self.excludelist_hostnames + ) + + def _after_request(self, span: Union[Span, BlankSpan], response: Response): + span.add_attribute(HTTP_STATUS_CODE, response.status_code) + + def _handle_exception(self, + span: Union[Span, BlankSpan], exception: Exception): + span.add_attribute(ERROR_NAME, exception.__class__.__name__) + span.add_attribute(ERROR_MESSAGE, str(exception)) + span.add_attribute( + STACKTRACE, + "\n".join(traceback.format_tb(exception.__traceback__))) + span.add_attribute(HTTP_STATUS_CODE, HTTP_500_INTERNAL_SERVER_ERROR) + + async def dispatch( + self, request: Request, call_next: RequestResponseEndpoint + ) -> Response: + + # Do not trace if the url is in the exclude list + if utils.disable_tracing_url(str(request.url), self.excludelist_paths): + return await call_next(request) + + try: + tracer = self._prepare_tracer(request) + span = tracer.start_span() + except Exception: # pragma: NO COVER + module_logger.error("Failed to trace request", exc_info=True) + return await call_next(request) + + try: + self._before_request(span, request) + except Exception: # pragma: NO COVER + module_logger.error("Failed to trace request", exc_info=True) + + try: + response = await call_next(request) + except Exception as err: # pragma: NO COVER + try: + self._handle_exception(span, err) + tracer.end_span() + tracer.finish() + except Exception: # pragma: NO COVER + module_logger.error("Failed to trace response", exc_info=True) + raise err + + try: + self._after_request(span, response) + tracer.end_span() + tracer.finish() + except Exception: # pragma: NO COVER + module_logger.error("Failed to trace response", exc_info=True) + + return response diff --git a/contrib/opencensus-ext-fastapi/setup.cfg b/contrib/opencensus-ext-fastapi/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/contrib/opencensus-ext-fastapi/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/contrib/opencensus-ext-fastapi/setup.py b/contrib/opencensus-ext-fastapi/setup.py new file mode 100644 index 000000000..f4ade731e --- /dev/null +++ b/contrib/opencensus-ext-fastapi/setup.py @@ -0,0 +1,49 @@ +# Copyright 2022, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import find_packages, setup + +from version import __version__ + +setup( + name='opencensus-ext-fastapi', + version=__version__, # noqa + author='OpenCensus Authors', + author_email='census-developers@googlegroups.com', + classifiers=[ + 'Intended Audience :: Developers', + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + ], + description='OpenCensus FastAPI Integration', + include_package_data=True, + long_description=open('README.rst').read(), + install_requires=[ + 'fastapi >= 0.75.2', + 'opencensus >= 0.9.dev0, < 1.0.0', + ], + extras_require={}, + license='Apache-2.0', + packages=find_packages(exclude=('tests',)), + namespace_packages=[], + url='https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-ext-fastapi', # noqa: E501 + zip_safe=False, +) diff --git a/contrib/opencensus-ext-fastapi/tests/test_fastapi_middleware.py b/contrib/opencensus-ext-fastapi/tests/test_fastapi_middleware.py new file mode 100644 index 000000000..e2d8f113f --- /dev/null +++ b/contrib/opencensus-ext-fastapi/tests/test_fastapi_middleware.py @@ -0,0 +1,197 @@ +# Copyright 2022, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import traceback +import unittest +from unittest.mock import ANY + +import mock +from fastapi import FastAPI +from starlette.testclient import TestClient + +from opencensus.ext.fastapi.fastapi_middleware import FastAPIMiddleware +from opencensus.trace import print_exporter, samplers +from opencensus.trace import span as span_module +from opencensus.trace import tracer as tracer_module +from opencensus.trace.propagation import trace_context_http_header_format + + +class FastAPITestException(Exception): + pass + + +class TestFastAPIMiddleware(unittest.TestCase): + + def tearDown(self) -> None: + from opencensus.trace import execution_context + execution_context.clear() + + return super().tearDown() + + def create_app(self): + app = FastAPI() + + @app.get('/') + def index(): + return 'test fastapi trace' # pragma: NO COVER + + @app.get('/wiki/{entry}') + def wiki(entry): + return 'test fastapi trace' # pragma: NO COVER + + @app.get('/health') + def health_check(): + return 'test health check' # pragma: NO COVER + + @app.get('/error') + def error(): + raise FastAPITestException('test error') + + return app + + def test_constructor_default(self): + app = self.create_app() + middleware = FastAPIMiddleware(app) + + self.assertIs(middleware.app, app) + self.assertIsNone(middleware.excludelist_paths) + self.assertIsNone(middleware.excludelist_hostnames) + self.assertIsInstance(middleware.sampler, samplers.AlwaysOnSampler) + self.assertIsInstance(middleware.exporter, + print_exporter.PrintExporter) + self.assertIsInstance( + middleware.propagator, + trace_context_http_header_format.TraceContextPropagator) + + def test_constructor_explicit(self): + excludelist_paths = mock.Mock() + excludelist_hostnames = mock.Mock() + sampler = mock.Mock() + exporter = mock.Mock() + propagator = mock.Mock() + + app = self.create_app() + middleware = FastAPIMiddleware( + app=app, + excludelist_paths=excludelist_paths, + excludelist_hostnames=excludelist_hostnames, + sampler=sampler, + exporter=exporter, + propagator=propagator) + + self.assertEqual(middleware.app, app) + self.assertEqual(middleware.excludelist_paths, excludelist_paths) + self.assertEqual( + middleware.excludelist_hostnames, excludelist_hostnames) + self.assertEqual(middleware.sampler, sampler) + self.assertEqual(middleware.exporter, exporter) + self.assertEqual(middleware.propagator, propagator) + + @mock.patch.object(tracer_module.Tracer, "finish") + @mock.patch.object(tracer_module.Tracer, "end_span") + @mock.patch.object(tracer_module.Tracer, "start_span") + def test_request(self, mock_m1, mock_m2, mock_m3): + app = self.create_app() + app.add_middleware( + FastAPIMiddleware, sampler=samplers.AlwaysOnSampler()) + + test_client = TestClient(app) + test_client.get("/wiki/Rabbit") + + mock_span = mock_m1.return_value + self.assertEqual(mock_span.add_attribute.call_count, 6) + mock_span.add_attribute.assert_has_calls([ + mock.call("http.host", "testserver"), + mock.call("http.method", "GET"), + mock.call("http.path", "/wiki/Rabbit"), + mock.call("http.url", "http://testserver/wiki/Rabbit"), + mock.call("http.route", "/wiki/Rabbit"), + mock.call("http.status_code", 200) + ]) + mock_m2.assert_called_once() + mock_m3.assert_called_once() + + self.assertEqual( + mock_span.span_kind, + span_module.SpanKind.SERVER) + self.assertEqual( + mock_span.name, + "[{}]{}".format("GET", "http://testserver/wiki/Rabbit")) + + @mock.patch.object(FastAPIMiddleware, "_prepare_tracer") + def test_request_excludelist(self, mock_m): + app = self.create_app() + app.add_middleware( + FastAPIMiddleware, + excludelist_paths=["health"], + sampler=samplers.AlwaysOnSampler()) + + test_client = TestClient(app) + test_client.get("/health") + + mock_m.assert_not_called() + + @mock.patch.object(tracer_module.Tracer, "finish") + @mock.patch.object(tracer_module.Tracer, "end_span") + @mock.patch.object(tracer_module.Tracer, "start_span") + def test_request_exception(self, mock_m1, mock_m2, mock_m3): + app = self.create_app() + app.add_middleware(FastAPIMiddleware) + + test_client = TestClient(app) + + with self.assertRaises(FastAPITestException): + test_client.get("/error") + + mock_span = mock_m1.return_value + self.assertEqual(mock_span.add_attribute.call_count, 9) + mock_span.add_attribute.assert_has_calls([ + mock.call("http.host", "testserver"), + mock.call("http.method", "GET"), + mock.call("http.path", "/error"), + mock.call("http.url", "http://testserver/error"), + mock.call("http.route", "/error"), + mock.call("error.name", "FastAPITestException"), + mock.call("error.message", "test error"), + mock.call("stacktrace", ANY), + mock.call("http.status_code", 500) + ]) + mock_m2.assert_called_once() + mock_m3.assert_called_once() + + def test_request_exception_stacktrace(self): + tb = None + try: + raise RuntimeError("bork bork bork") + except Exception as exc: + test_exception = exc + if hasattr(exc, "__traceback__"): + tb = exc.__traceback__ + else: + _, _, tb = sys.exc_info() + + app = self.create_app() + middleware = FastAPIMiddleware(app) + + mock_span = mock.Mock() + mock_span.add_attribute = mock.Mock() + middleware._handle_exception(mock_span, test_exception) + + mock_span.add_attribute.assert_has_calls([ + mock.call("error.name", "RuntimeError"), + mock.call("error.message", "bork bork bork"), + mock.call("stacktrace", "\n".join(traceback.format_tb(tb))), + mock.call("http.status_code", 500) + ]) diff --git a/contrib/opencensus-ext-fastapi/version.py b/contrib/opencensus-ext-fastapi/version.py new file mode 100644 index 000000000..50e4d191e --- /dev/null +++ b/contrib/opencensus-ext-fastapi/version.py @@ -0,0 +1,15 @@ +# Copyright 2022, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = '0.2.dev0' diff --git a/contrib/opencensus-ext-flask/CHANGELOG.md b/contrib/opencensus-ext-flask/CHANGELOG.md index 498fd8eeb..1218b195d 100644 --- a/contrib/opencensus-ext-flask/CHANGELOG.md +++ b/contrib/opencensus-ext-flask/CHANGELOG.md @@ -1,7 +1,63 @@ # Changelog ## Unreleased + +## 0.8.2 +Released 2023-03-10 + +- Add exception information to span attributes +([#1188](https://github.com/census-instrumentation/opencensus-python/pull/1188)) + +## 0.8.1 +Released 2022-08-03 + +- Move `version.py` file into `common` folder +([#1143](https://github.com/census-instrumentation/opencensus-python/pull/1143)) + +## 0.8.0 +Released 2022-04-20 + +- Add support for Flask 2 +([#1032](https://github.com/census-instrumentation/opencensus-python/pull/1032)) + +## 0.7.5 +Released 2021-05-13 + +- Restrict `flask` version `1.1.3` as it throws an exception on `Python v2.7` - https://github.com/pallets/flask/issues/4050 +([#1032](https://github.com/census-instrumentation/opencensus-python/pull/1032)) + +## 0.7.4 +Released 2021-01-14 + +- Change blacklist to excludelist +([#977](https://github.com/census-instrumentation/opencensus-python/pull/977)) + +## 0.7.3 +Released 2019-10-01 + +- Check that `url_rule` is not `None` before dereferencing property. + ([#781](https://github.com/census-instrumentation/opencensus-python/pull/781)) + +## 0.7.2 +Released 2019-08-26 + +- Updated `http.status_code` attribute to be an int. + ([#755](https://github.com/census-instrumentation/opencensus-python/pull/755)) +- Fixes value for `http.route` in Flask middleware + ([#759](https://github.com/census-instrumentation/opencensus-python/pull/759)) + +## 0.7.1 +Released 2019-08-05 + +- Update for core library changes + +## 0.7.0 +Released 2019-07-31 + - Make ProbabilitySampler default +- Updated span attributes to include some missing attributes listed + [here](https://github.com/census-instrumentation/opencensus-specs/blob/master/trace/HTTP.md#attributes) + ([#735](https://github.com/census-instrumentation/opencensus-python/pull/735)) ## 0.3.0 Released 2019-04-24 diff --git a/contrib/opencensus-ext-flask/README.rst b/contrib/opencensus-ext-flask/README.rst index 797e2bbbc..1252f291a 100644 --- a/contrib/opencensus-ext-flask/README.rst +++ b/contrib/opencensus-ext-flask/README.rst @@ -22,7 +22,7 @@ Usage from opencensus.ext.flask.flask_middleware import FlaskMiddleware app = Flask(__name__) - middleware = FlaskMiddleware(app, blacklist_paths=['_ah/health']) + middleware = FlaskMiddleware(app, excludelist_paths=['_ah/health']) @app.route('/') def hello(): diff --git a/contrib/opencensus-ext-flask/examples/custom.py b/contrib/opencensus-ext-flask/examples/custom.py index c310958a4..d6f8fd86d 100644 --- a/contrib/opencensus-ext-flask/examples/custom.py +++ b/contrib/opencensus-ext-flask/examples/custom.py @@ -21,13 +21,12 @@ import requests import sqlalchemy +import hello_world_pb2 +import hello_world_pb2_grpc from opencensus.ext.flask.flask_middleware import FlaskMiddleware from opencensus.ext.grpc import client_interceptor from opencensus.ext.stackdriver import trace_exporter as stackdriver_exporter -from opencensus.trace import config_integration -from opencensus.trace import samplers -import hello_world_pb2 -import hello_world_pb2_grpc +from opencensus.trace import config_integration, samplers INTEGRATIONS = ['mysql', 'postgresql', 'sqlalchemy', 'requests'] diff --git a/contrib/opencensus-ext-flask/examples/simple.py b/contrib/opencensus-ext-flask/examples/simple.py index ac4eb50e9..0f3ca65c0 100644 --- a/contrib/opencensus-ext-flask/examples/simple.py +++ b/contrib/opencensus-ext-flask/examples/simple.py @@ -13,6 +13,7 @@ # limitations under the License. from flask import Flask + from opencensus.ext.flask.flask_middleware import FlaskMiddleware app = Flask(__name__) diff --git a/contrib/opencensus-ext-flask/opencensus/ext/flask/common/__init__.py b/contrib/opencensus-ext-flask/opencensus/ext/flask/common/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-flask/opencensus/ext/flask/common/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-flask/version.py b/contrib/opencensus-ext-flask/opencensus/ext/flask/common/version.py similarity index 95% rename from contrib/opencensus-ext-flask/version.py rename to contrib/opencensus-ext-flask/opencensus/ext/flask/common/version.py index deb2f374d..671fc3d04 100644 --- a/contrib/opencensus-ext-flask/version.py +++ b/contrib/opencensus-ext-flask/opencensus/ext/flask/common/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.4.dev0' +__version__ = '0.9.dev0' diff --git a/contrib/opencensus-ext-flask/opencensus/ext/flask/flask_middleware.py b/contrib/opencensus-ext-flask/opencensus/ext/flask/flask_middleware.py index 328b53f10..08ccc10ab 100644 --- a/contrib/opencensus-ext-flask/opencensus/ext/flask/flask_middleware.py +++ b/contrib/opencensus-ext-flask/opencensus/ext/flask/flask_middleware.py @@ -12,31 +12,41 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import six + +import logging import sys +import traceback import flask from google.rpc import code_pb2 from opencensus.common import configuration -from opencensus.trace import attributes_helper -from opencensus.trace import execution_context -from opencensus.trace import print_exporter -from opencensus.trace import samplers +from opencensus.trace import ( + attributes_helper, + execution_context, + integrations, + print_exporter, + samplers, +) from opencensus.trace import span as span_module -from opencensus.trace import stack_trace -from opencensus.trace import status +from opencensus.trace import stack_trace, status from opencensus.trace import tracer as tracer_module from opencensus.trace import utils from opencensus.trace.propagation import trace_context_http_header_format +HTTP_HOST = attributes_helper.COMMON_ATTRIBUTES['HTTP_HOST'] HTTP_METHOD = attributes_helper.COMMON_ATTRIBUTES['HTTP_METHOD'] +HTTP_PATH = attributes_helper.COMMON_ATTRIBUTES['HTTP_PATH'] +HTTP_ROUTE = attributes_helper.COMMON_ATTRIBUTES['HTTP_ROUTE'] HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL'] HTTP_STATUS_CODE = attributes_helper.COMMON_ATTRIBUTES['HTTP_STATUS_CODE'] +ERROR_MESSAGE = attributes_helper.COMMON_ATTRIBUTES['ERROR_MESSAGE'] +ERROR_NAME = attributes_helper.COMMON_ATTRIBUTES['ERROR_NAME'] +STACKTRACE = attributes_helper.COMMON_ATTRIBUTES['STACKTRACE'] -BLACKLIST_PATHS = 'BLACKLIST_PATHS' -BLACKLIST_HOSTNAMES = 'BLACKLIST_HOSTNAMES' +EXCLUDELIST_PATHS = 'EXCLUDELIST_PATHS' +EXCLUDELIST_HOSTNAMES = 'EXCLUDELIST_HOSTNAMES' log = logging.getLogger(__name__) @@ -47,8 +57,8 @@ class FlaskMiddleware(object): :type app: :class: `~flask.Flask` :param app: A flask application. - :type blacklist_paths: list - :param blacklist_paths: Paths that do not trace. + :type excludelist_paths: list + :param excludelist_paths: Paths that do not trace. :type sampler: :class:`~opencensus.trace.samplers.base.Sampler` :param sampler: A sampler. It should extend from the base @@ -71,10 +81,10 @@ class FlaskMiddleware(object): :class:`.TextFormatPropagator`. """ - def __init__(self, app=None, blacklist_paths=None, sampler=None, + def __init__(self, app=None, excludelist_paths=None, sampler=None, exporter=None, propagator=None): self.app = app - self.blacklist_paths = blacklist_paths + self.excludelist_paths = excludelist_paths self.sampler = sampler self.exporter = exporter self.propagator = propagator @@ -107,13 +117,16 @@ def init_app(self, app): if isinstance(self.propagator, six.string_types): self.propagator = configuration.load(self.propagator) - self.blacklist_paths = settings.get(BLACKLIST_PATHS, - self.blacklist_paths) + self.excludelist_paths = settings.get(EXCLUDELIST_PATHS, + self.excludelist_paths) - self.blacklist_hostnames = settings.get(BLACKLIST_HOSTNAMES, None) + self.excludelist_hostnames = settings.get(EXCLUDELIST_HOSTNAMES, None) self.setup_trace() + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.FLASK) + def setup_trace(self): self.app.before_request(self._before_request) self.app.after_request(self._after_request) @@ -124,8 +137,10 @@ def _before_request(self): See: http://flask.pocoo.org/docs/0.12/api/#flask.Flask.before_request """ - # Do not trace if the url is blacklisted - if utils.disable_tracing_url(flask.request.url, self.blacklist_paths): + # Do not trace if the url is in the exclude list + if utils.disable_tracing_url( + flask.request.url, self.excludelist_paths + ): return try: @@ -144,12 +159,21 @@ def _before_request(self): flask.request.method, flask.request.url) tracer.add_attribute_to_current_span( - HTTP_METHOD, flask.request.method) + HTTP_HOST, flask.request.host + ) tracer.add_attribute_to_current_span( - HTTP_URL, str(flask.request.url)) + HTTP_METHOD, flask.request.method + ) + tracer.add_attribute_to_current_span( + HTTP_PATH, flask.request.path + ) + tracer.add_attribute_to_current_span( + HTTP_URL, str(flask.request.url) + ) execution_context.set_opencensus_attr( - 'blacklist_hostnames', - self.blacklist_hostnames) + 'excludelist_hostnames', + self.excludelist_hostnames + ) except Exception: # pragma: NO COVER log.error('Failed to trace request', exc_info=True) @@ -158,23 +182,33 @@ def _after_request(self, response): See: http://flask.pocoo.org/docs/0.12/api/#flask.Flask.after_request """ - # Do not trace if the url is blacklisted - if utils.disable_tracing_url(flask.request.url, self.blacklist_paths): + # Do not trace if the url is in the exclude list + if utils.disable_tracing_url( + flask.request.url, self.excludelist_paths + ): return response try: tracer = execution_context.get_opencensus_tracer() + url_rule = flask.request.url_rule + if url_rule is not None: + tracer.add_attribute_to_current_span( + HTTP_ROUTE, url_rule.rule + ) tracer.add_attribute_to_current_span( HTTP_STATUS_CODE, - str(response.status_code)) + response.status_code + ) except Exception: # pragma: NO COVER log.error('Failed to trace request', exc_info=True) finally: return response def _teardown_request(self, exception): - # Do not trace if the url is blacklisted - if utils.disable_tracing_url(flask.request.url, self.blacklist_paths): + # Do not trace if the url is in the exclude list + if utils.disable_tracing_url( + flask.request.url, self.excludelist_paths + ): return try: @@ -187,16 +221,28 @@ def _teardown_request(self, exception): code=code_pb2.UNKNOWN, message=str(exception) ) - # try attaching the stack trace to the span, only populated - # if the app has 'PROPAGATE_EXCEPTIONS', 'DEBUG', or - # 'TESTING' enabled - exc_type, _, exc_traceback = sys.exc_info() + span.add_attribute( + attribute_key=ERROR_NAME, + attribute_value=exception.__class__.__name__) + span.add_attribute( + attribute_key=ERROR_MESSAGE, + attribute_value=str(exception)) + + if hasattr(exception, '__traceback__'): + exc_traceback = exception.__traceback__ + else: + exc_type, _, exc_traceback = sys.exc_info() if exc_traceback is not None: span.stack_trace = ( stack_trace.StackTrace.from_traceback( exc_traceback ) ) + span.add_attribute( + attribute_key=STACKTRACE, + attribute_value='\n'.join( + traceback.format_tb(exc_traceback)) + ) tracer.end_span() tracer.finish() diff --git a/contrib/opencensus-ext-flask/setup.py b/contrib/opencensus-ext-flask/setup.py index a1d6db8b1..63aa51329 100644 --- a/contrib/opencensus-ext-flask/setup.py +++ b/contrib/opencensus-ext-flask/setup.py @@ -12,13 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup -from version import __version__ +import os + +from setuptools import find_packages, setup + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, "opencensus", "ext", "flask", "common", "version.py" +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) setup( name='opencensus-ext-flask', - version=__version__, # noqa + version=PACKAGE_INFO["__version__"], # noqa author='OpenCensus Authors', author_email='census-developers@googlegroups.com', classifiers=[ @@ -34,13 +42,15 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Flask Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'flask >= 0.12.3, < 2.0.0', - 'opencensus >= 0.7.dev0, < 1.0.0', + 'flask >= 0.12.3, < 3.0.0, != 1.1.3', + 'opencensus >= 0.12.dev0, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-flask/tests/test_flask_middleware.py b/contrib/opencensus-ext-flask/tests/test_flask_middleware.py index 1855e721f..218acab34 100644 --- a/contrib/opencensus-ext-flask/tests/test_flask_middleware.py +++ b/contrib/opencensus-ext-flask/tests/test_flask_middleware.py @@ -17,24 +17,20 @@ import unittest -from google.rpc import code_pb2 import flask import mock +from google.rpc import code_pb2 +from werkzeug.exceptions import NotFound from opencensus.ext.flask import flask_middleware -from opencensus.trace import execution_context -from opencensus.trace import print_exporter -from opencensus.trace import samplers +from opencensus.trace import execution_context, print_exporter, samplers from opencensus.trace import span as span_module -from opencensus.trace import span_data -from opencensus.trace import stack_trace -from opencensus.trace import status +from opencensus.trace import span_data, stack_trace, status from opencensus.trace.blank_span import BlankSpan from opencensus.trace.propagation import trace_context_http_header_format from opencensus.trace.span_context import SpanContext from opencensus.trace.trace_options import TraceOptions -from opencensus.trace.tracers import base -from opencensus.trace.tracers import noop_tracer +from opencensus.trace.tracers import base, noop_tracer class FlaskTestException(Exception): @@ -51,6 +47,10 @@ def create_app(): def index(): return 'test flask trace' # pragma: NO COVER + @app.route('/wiki/') + def wiki(entry): + return 'test flask trace' # pragma: NO COVER + @app.route('/_ah/health') def health_check(): return 'test health check' # pragma: NO COVER @@ -141,7 +141,7 @@ def test__before_request(self): flask_middleware.FlaskMiddleware(app=app, sampler=samplers.AlwaysOnSampler()) context = app.test_request_context( - path='/', + path='/wiki/Rabbit', headers={flask_trace_header: flask_trace_id}) with context: @@ -152,8 +152,10 @@ def test__before_request(self): span = tracer.current_span() expected_attributes = { - 'http.url': u'http://localhost/', - 'http.method': 'GET', + 'http.host': u'localhost', + 'http.method': u'GET', + 'http.path': u'/wiki/Rabbit', + 'http.url': u'http://localhost/wiki/Rabbit', } self.assertEqual(span.span_kind, span_module.SpanKind.SERVER) @@ -163,14 +165,14 @@ def test__before_request(self): span_context = tracer.span_context self.assertEqual(span_context.trace_id, trace_id) - def test__before_request_blacklist(self): + def test__before_request_excludelist(self): flask_trace_header = 'traceparent' trace_id = '2dd43a1d6b2549c6bc2a1a54c2fc0b05' span_id = '6e0c63257de34c92' flask_trace_id = '00-{}-{}-00'.format(trace_id, span_id) app = self.create_app() - # Use the AlwaysOnSampler here to prove that the blacklist takes + # Use the AlwaysOnSampler here to prove that the excludelist takes # precedence over the sampler flask_middleware.FlaskMiddleware(app=app, sampler=samplers.AlwaysOnSampler()) @@ -203,7 +205,7 @@ def test_header_encoding(self): flask_middleware.FlaskMiddleware(app=app, sampler=samplers.AlwaysOnSampler()) context = app.test_request_context( - path='/', + path='/wiki/Rabbit', headers={flask_trace_header: flask_trace_id}) with context: @@ -214,8 +216,10 @@ def test_header_encoding(self): span = tracer.current_span() expected_attributes = { - 'http.url': u'http://localhost/', - 'http.method': 'GET', + 'http.host': u'localhost', + 'http.method': u'GET', + 'http.path': u'/wiki/Rabbit', + 'http.url': u'http://localhost/wiki/Rabbit', } self.assertEqual(span.attributes, expected_attributes) @@ -229,7 +233,7 @@ def test_header_is_none(self): flask_middleware.FlaskMiddleware(app=app, sampler=samplers.AlwaysOnSampler()) context = app.test_request_context( - path='/') + path='/wiki/Rabbit') with context: app.preprocess_request() @@ -239,8 +243,10 @@ def test_header_is_none(self): span = tracer.current_span() expected_attributes = { - 'http.url': u'http://localhost/', - 'http.method': 'GET', + 'http.host': u'localhost', + 'http.method': u'GET', + 'http.path': u'/wiki/Rabbit', + 'http.url': u'http://localhost/wiki/Rabbit', } self.assertEqual(span.attributes, expected_attributes) @@ -269,15 +275,81 @@ def test__after_request_sampled(self): flask_trace_id = '00-{}-{}-00'.format(trace_id, span_id) app = self.create_app() - flask_middleware.FlaskMiddleware(app=app) + flask_middleware.FlaskMiddleware( + app=app, + sampler=samplers.AlwaysOnSampler() + ) - response = app.test_client().get( - '/', - headers={flask_trace_header: flask_trace_id}) + context = app.test_request_context( + path='/wiki/Rabbit', + headers={flask_trace_header: flask_trace_id} + ) - self.assertEqual(response.status_code, 200) + with context: + app.preprocess_request() + tracer = execution_context.get_opencensus_tracer() + self.assertIsNotNone(tracer) + + span = tracer.current_span() + + rv = app.dispatch_request() + app.finalize_request(rv) + + expected_attributes = { + 'http.host': u'localhost', + 'http.method': u'GET', + 'http.path': u'/wiki/Rabbit', + 'http.url': u'http://localhost/wiki/Rabbit', + 'http.route': u'/wiki/', + 'http.status_code': 200 + } + + self.assertEqual(span.attributes, expected_attributes) + assert isinstance(span.parent_span, base.NullContextManager) + + def test__after_request_invalid_url(self): + flask_trace_header = 'traceparent' + trace_id = '2dd43a1d6b2549c6bc2a1a54c2fc0b05' + span_id = '6e0c63257de34c92' + flask_trace_id = '00-{}-{}-00'.format(trace_id, span_id) + + app = self.create_app() + flask_middleware.FlaskMiddleware( + app=app, + sampler=samplers.AlwaysOnSampler() + ) + + context = app.test_request_context( + path='/this-url-does-not-exist', + headers={flask_trace_header: flask_trace_id} + ) + + with context: + app.preprocess_request() + tracer = execution_context.get_opencensus_tracer() + self.assertIsNotNone(tracer) + + span = tracer.current_span() + + try: + rv = app.dispatch_request() + except NotFound as e: + rv = app.handle_user_exception(e) + app.finalize_request(rv) + + # http.route should not be set + expected_attributes = { + 'http.host': u'localhost', + 'http.method': u'GET', + 'http.path': u'/this-url-does-not-exist', + 'http.url': u'http://localhost/this-url-does-not-exist', + 'http.status_code': 404 + } - def test__after_request_blacklist(self): + self.assertEqual(span.attributes, expected_attributes) + assert isinstance(span.parent_span, base.NullContextManager) + + def test__after_request_excludelist(self): flask_trace_header = 'traceparent' trace_id = '2dd43a1d6b2549c6bc2a1a54c2fc0b05' span_id = '6e0c63257de34c92' @@ -307,8 +379,17 @@ def test_teardown_include_exception(self): exported_spandata = mock_exporter.export.call_args[0][0][0] self.assertIsInstance(exported_spandata, span_data.SpanData) self.assertIsInstance(exported_spandata.status, status.Status) - self.assertEqual(exported_spandata.status.code, code_pb2.UNKNOWN) - self.assertEqual(exported_spandata.status.message, 'error') + self.assertEqual( + exported_spandata.status.canonical_code, code_pb2.UNKNOWN + ) + self.assertEqual(exported_spandata.status.description, 'error') + self.assertEqual( + exported_spandata.attributes["error.name"], 'FlaskTestException' + ) + self.assertEqual( + exported_spandata.attributes["error.message"], 'error' + ) + self.assertIsNotNone(exported_spandata.attributes["error.message"]) def test_teardown_include_exception_and_traceback(self): mock_exporter = mock.MagicMock() @@ -322,8 +403,10 @@ def test_teardown_include_exception_and_traceback(self): exported_spandata = mock_exporter.export.call_args[0][0][0] self.assertIsInstance(exported_spandata, span_data.SpanData) self.assertIsInstance(exported_spandata.status, status.Status) - self.assertEqual(exported_spandata.status.code, code_pb2.UNKNOWN) - self.assertEqual(exported_spandata.status.message, 'error') + self.assertEqual( + exported_spandata.status.canonical_code, code_pb2.UNKNOWN + ) + self.assertEqual(exported_spandata.status.description, 'error') self.assertIsInstance( exported_spandata.stack_trace, stack_trace.StackTrace ) diff --git a/contrib/opencensus-ext-gevent/opencensus/ext/gevent/geventcompatibility.py b/contrib/opencensus-ext-gevent/opencensus/ext/gevent/geventcompatibility.py index f1f662c48..d59633573 100644 --- a/contrib/opencensus-ext-gevent/opencensus/ext/gevent/geventcompatibility.py +++ b/contrib/opencensus-ext-gevent/opencensus/ext/gevent/geventcompatibility.py @@ -1,4 +1,5 @@ import logging + import gevent.monkey diff --git a/contrib/opencensus-ext-gevent/setup.py b/contrib/opencensus-ext-gevent/setup.py index 4109854aa..64fedd231 100644 --- a/contrib/opencensus-ext-gevent/setup.py +++ b/contrib/opencensus-ext-gevent/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus gevent compatibility helper', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', 'gevent >= 1.3' ], extras_require={}, diff --git a/contrib/opencensus-ext-gevent/tests/test_patching.py b/contrib/opencensus-ext-gevent/tests/test_patching.py index 08955fdd8..5172d54f7 100644 --- a/contrib/opencensus-ext-gevent/tests/test_patching.py +++ b/contrib/opencensus-ext-gevent/tests/test_patching.py @@ -1,96 +1,96 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# # Copyright 2019, OpenCensus Authors +# # +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. -import unittest +# import unittest -import opencensus.common.runtime_context as runtime_context -import gevent.monkey +# import gevent.monkey +# import mock -import mock +# import opencensus.common.runtime_context as runtime_context -class TestPatching(unittest.TestCase): - def setUp(self): - self.original_context = runtime_context.RuntimeContext +# class TestPatching(unittest.TestCase): +# def setUp(self): +# self.original_context = runtime_context.RuntimeContext - def tearDown(self): - runtime_context.RuntimeContext = self.original_context +# def tearDown(self): +# runtime_context.RuntimeContext = self.original_context - @mock.patch("gevent.monkey.is_module_patched", return_value=False) - def test_context_is_switched_without_contextvar_support( - self, patched_is_module_patched - ): - # patched_is_module_patched.return_value = False +# @mock.patch("gevent.monkey.is_module_patched", return_value=False) +# def test_context_is_switched_without_contextvar_support( +# self, patched_is_module_patched +# ): +# # patched_is_module_patched.return_value = False - # Trick gevent into thinking it is run for the first time. - # Allows to run multiple tests. - gevent.monkey.saved = {} +# # Trick gevent into thinking it is run for the first time. +# # Allows to run multiple tests. +# gevent.monkey.saved = {} - # All module patching is disabled to avoid the need of "unpatching". - # The needed events are emitted nevertheless. - gevent.monkey.patch_all( - contextvar=False, - socket=False, - dns=False, - time=False, - select=False, - thread=False, - os=False, - ssl=False, - httplib=False, - subprocess=False, - sys=False, - aggressive=False, - Event=False, - builtins=False, - signal=False, - queue=False - ) +# # All module patching is disabled to avoid the need of "unpatching". +# # The needed events are emitted nevertheless. +# gevent.monkey.patch_all( +# contextvar=False, +# socket=False, +# dns=False, +# time=False, +# select=False, +# thread=False, +# os=False, +# ssl=False, +# httplib=False, +# subprocess=False, +# sys=False, +# aggressive=False, +# Event=False, +# builtins=False, +# signal=False, +# queue=False +# ) - assert isinstance( - runtime_context.RuntimeContext, - runtime_context._ThreadLocalRuntimeContext, - ) +# assert isinstance( +# runtime_context.RuntimeContext, +# runtime_context._ThreadLocalRuntimeContext, +# ) - @mock.patch("gevent.monkey.is_module_patched", return_value=True) - def test_context_is_switched_with_contextvar_support( - self, patched_is_module_patched - ): +# @mock.patch("gevent.monkey.is_module_patched", return_value=True) +# def test_context_is_switched_with_contextvar_support( +# self, patched_is_module_patched +# ): - # Trick gevent into thinking it is run for the first time. - # Allows to run multiple tests. - gevent.monkey.saved = {} +# # Trick gevent into thinking it is run for the first time. +# # Allows to run multiple tests. +# gevent.monkey.saved = {} - # All module patching is disabled to avoid the need of "unpatching". - # The needed events are emitted nevertheless. - gevent.monkey.patch_all( - contextvar=False, - socket=False, - dns=False, - time=False, - select=False, - thread=False, - os=False, - ssl=False, - httplib=False, - subprocess=False, - sys=False, - aggressive=False, - Event=False, - builtins=False, - signal=False, - queue=False - ) +# # All module patching is disabled to avoid the need of "unpatching". +# # The needed events are emitted nevertheless. +# gevent.monkey.patch_all( +# contextvar=False, +# socket=False, +# dns=False, +# time=False, +# select=False, +# thread=False, +# os=False, +# ssl=False, +# httplib=False, +# subprocess=False, +# sys=False, +# aggressive=False, +# Event=False, +# builtins=False, +# signal=False, +# queue=False +# ) - assert runtime_context.RuntimeContext is self.original_context +# assert runtime_context.RuntimeContext is self.original_context diff --git a/contrib/opencensus-ext-google-cloud-clientlibs/CHANGELOG.md b/contrib/opencensus-ext-google-cloud-clientlibs/CHANGELOG.md index f94e0621b..02a79ac86 100644 --- a/contrib/opencensus-ext-google-cloud-clientlibs/CHANGELOG.md +++ b/contrib/opencensus-ext-google-cloud-clientlibs/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +## 0.1.3 +Released 2021-10-04 +- Added missing google-cloud-core dependency + [#1049](https://github.com/census-instrumentation/opencensus-python/pull/1049) + ## 0.1.2 Released 2019-04-24 diff --git a/contrib/opencensus-ext-google-cloud-clientlibs/opencensus/ext/google_cloud_clientlibs/trace.py b/contrib/opencensus-ext-google-cloud-clientlibs/opencensus/ext/google_cloud_clientlibs/trace.py index 3bc9f8e71..9c3741aa9 100644 --- a/contrib/opencensus-ext-google-cloud-clientlibs/opencensus/ext/google_cloud_clientlibs/trace.py +++ b/contrib/opencensus-ext-google-cloud-clientlibs/opencensus/ext/google_cloud_clientlibs/trace.py @@ -15,15 +15,12 @@ import logging import grpc - -from google.cloud import _helpers from google.api_core import grpc_helpers +from google.cloud import _helpers -from opencensus.ext.grpc.client_interceptor import ( - OpenCensusClientInterceptor) - -from opencensus.ext.requests.trace import ( - trace_integration as trace_requests) +from opencensus.ext.grpc.client_interceptor import OpenCensusClientInterceptor +from opencensus.ext.requests.trace import trace_integration as trace_requests +from opencensus.trace import integrations log = logging.getLogger(__name__) @@ -46,6 +43,9 @@ def trace_integration(tracer=None): # Integrate with HTTP trace_http(tracer) + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.GOOGLE_CLOUD) + def trace_grpc(tracer=None): """Integrate with gRPC.""" diff --git a/contrib/opencensus-ext-google-cloud-clientlibs/setup.py b/contrib/opencensus-ext-google-cloud-clientlibs/setup.py index 2dc87bd9e..5768b1cf5 100644 --- a/contrib/opencensus-ext-google-cloud-clientlibs/setup.py +++ b/contrib/opencensus-ext-google-cloud-clientlibs/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,14 +34,18 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Google Cloud Client Libraries Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', - 'opencensus-ext-grpc >= 0.4.dev0, < 1.0.0', - 'opencensus-ext-requests >= 0.2.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', + 'opencensus-ext-grpc >= 0.3.0, < 1.0.0', + 'opencensus-ext-requests >= 0.1.2, < 1.0.0', + 'google-cloud-core ~= 1.7; python_version == "2.7" or python_version >= "3.6"', # noqa: E501 + 'google-cloud-core ~= 1.4; python_version == "3.4" or python_version == "3.5"', # noqa: E501 ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-google-cloud-clientlibs/tests/test_google_cloud_clientlibs_trace.py b/contrib/opencensus-ext-google-cloud-clientlibs/tests/test_google_cloud_clientlibs_trace.py index fa10ba9f5..c503fadff 100644 --- a/contrib/opencensus-ext-google-cloud-clientlibs/tests/test_google_cloud_clientlibs_trace.py +++ b/contrib/opencensus-ext-google-cloud-clientlibs/tests/test_google_cloud_clientlibs_trace.py @@ -14,8 +14,8 @@ import unittest -import mock import grpc +import mock from opencensus.ext.google_cloud_clientlibs import trace diff --git a/contrib/opencensus-ext-grpc/CHANGELOG.md b/contrib/opencensus-ext-grpc/CHANGELOG.md index 33aafa3d6..0562d9216 100644 --- a/contrib/opencensus-ext-grpc/CHANGELOG.md +++ b/contrib/opencensus-ext-grpc/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +## 0.7.2 +Released 2021-01-14 + +- Extract byte size from proto-plus messages + ([#976](https://github.com/census-instrumentation/opencensus-python/pull/976)) + +## 0.7.1 +Released 2019-08-05 + +- Update for core library changes + ## 0.3.0 Released 2019-05-31 diff --git a/contrib/opencensus-ext-grpc/README.rst b/contrib/opencensus-ext-grpc/README.rst index 29ebc4007..5a84d8bc5 100644 --- a/contrib/opencensus-ext-grpc/README.rst +++ b/contrib/opencensus-ext-grpc/README.rst @@ -31,14 +31,12 @@ Installation Usage ----- -.. code:: python - - from opencensus.trace import config_integration - - config_integration.trace_integrations(['grpc']) +Please refer to the `examples `_. +For more information, read `gRPC Python Interceptors `_. References ---------- +* `Examples `_ * `gRPC `_ * `OpenCensus Project `_ diff --git a/contrib/opencensus-ext-grpc/examples/hello_world_client.py b/contrib/opencensus-ext-grpc/examples/hello_world_client.py index 48e662118..1875c35bc 100644 --- a/contrib/opencensus-ext-grpc/examples/hello_world_client.py +++ b/contrib/opencensus-ext-grpc/examples/hello_world_client.py @@ -18,10 +18,9 @@ import hello_world_pb2 import hello_world_pb2_grpc - -from opencensus.trace.tracer import Tracer from opencensus.ext.grpc import client_interceptor from opencensus.ext.stackdriver import trace_exporter as stackdriver_exporter +from opencensus.trace.tracer import Tracer HOST_PORT = 'localhost:50051' diff --git a/contrib/opencensus-ext-grpc/examples/hello_world_pb2.py b/contrib/opencensus-ext-grpc/examples/hello_world_pb2.py index 5927b2a1a..b67e7f718 100644 --- a/contrib/opencensus-ext-grpc/examples/hello_world_pb2.py +++ b/contrib/opencensus-ext-grpc/examples/hello_world_pb2.py @@ -3,12 +3,14 @@ # source: hello_world.proto import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) + from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pb2 from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 + +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() diff --git a/contrib/opencensus-ext-grpc/examples/hello_world_server.py b/contrib/opencensus-ext-grpc/examples/hello_world_server.py index 4698e3569..c7c559554 100644 --- a/contrib/opencensus-ext-grpc/examples/hello_world_server.py +++ b/contrib/opencensus-ext-grpc/examples/hello_world_server.py @@ -12,14 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from concurrent import futures import time +from concurrent import futures import grpc import hello_world_pb2 import hello_world_pb2_grpc - from opencensus.ext.grpc import server_interceptor from opencensus.ext.stackdriver import trace_exporter as stackdriver_exporter from opencensus.trace import samplers diff --git a/contrib/opencensus-ext-grpc/opencensus/ext/grpc/client_interceptor.py b/contrib/opencensus-ext-grpc/opencensus/ext/grpc/client_interceptor.py index 4fa9ccdf5..288bf650b 100644 --- a/contrib/opencensus-ext-grpc/opencensus/ext/grpc/client_interceptor.py +++ b/contrib/opencensus-ext-grpc/opencensus/ext/grpc/client_interceptor.py @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import six + import collections import logging import grpc -import six from opencensus.ext import grpc as oc_grpc from opencensus.ext.grpc import utils as grpc_utils -from opencensus.trace import attributes_helper -from opencensus.trace import execution_context +from opencensus.trace import attributes_helper, execution_context from opencensus.trace import span as span_module from opencensus.trace import time_event from opencensus.trace.propagation import binary_format diff --git a/contrib/opencensus-ext-grpc/opencensus/ext/grpc/server_interceptor.py b/contrib/opencensus-ext-grpc/opencensus/ext/grpc/server_interceptor.py index 2308e3dea..e8ffd6b91 100644 --- a/contrib/opencensus-ext-grpc/opencensus/ext/grpc/server_interceptor.py +++ b/contrib/opencensus-ext-grpc/opencensus/ext/grpc/server_interceptor.py @@ -18,20 +18,47 @@ from opencensus.ext import grpc as oc_grpc from opencensus.ext.grpc import utils as grpc_utils -from opencensus.trace import attributes_helper -from opencensus.trace import execution_context +from opencensus.trace import attributes_helper, execution_context from opencensus.trace import span as span_module from opencensus.trace import stack_trace as stack_trace -from opencensus.trace import status -from opencensus.trace import time_event +from opencensus.trace import status, time_event from opencensus.trace import tracer as tracer_module from opencensus.trace.propagation import binary_format -ATTRIBUTE_COMPONENT = 'COMPONENT' -ATTRIBUTE_ERROR_NAME = 'ERROR_NAME' -ATTRIBUTE_ERROR_MESSAGE = 'ERROR_MESSAGE' +COMPONENT = attributes_helper.COMMON_ATTRIBUTES['COMPONENT'] +ERROR_NAME = attributes_helper.COMMON_ATTRIBUTES['ERROR_NAME'] +ERROR_MESSAGE = attributes_helper.COMMON_ATTRIBUTES['ERROR_MESSAGE'] + +HTTP_HOST = attributes_helper.COMMON_ATTRIBUTES['HTTP_HOST'] +HTTP_METHOD = attributes_helper.COMMON_ATTRIBUTES['HTTP_METHOD'] +HTTP_PATH = attributes_helper.COMMON_ATTRIBUTES['HTTP_PATH'] +HTTP_ROUTE = attributes_helper.COMMON_ATTRIBUTES['HTTP_ROUTE'] +HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL'] +HTTP_STATUS_CODE = attributes_helper.COMMON_ATTRIBUTES['HTTP_STATUS_CODE'] +GRPC_METHOD = attributes_helper.GRPC_ATTRIBUTES['GRPC_METHOD'] + RECV_PREFIX = 'Recv' +GRPC_HTTP_STATUS_MAPPING = { + grpc.StatusCode.OK: 200, + grpc.StatusCode.FAILED_PRECONDITION: 400, + grpc.StatusCode.INVALID_ARGUMENT: 400, + grpc.StatusCode.OUT_OF_RANGE: 400, + grpc.StatusCode.UNAUTHENTICATED: 401, + grpc.StatusCode.PERMISSION_DENIED: 403, + grpc.StatusCode.NOT_FOUND: 404, + grpc.StatusCode.ABORTED: 409, + grpc.StatusCode.ALREADY_EXISTS: 409, + grpc.StatusCode.RESOURCE_EXHAUSTED: 429, + grpc.StatusCode.CANCELLED: 499, + grpc.StatusCode.UNKNOWN: 500, + grpc.StatusCode.INTERNAL: 500, + grpc.StatusCode.DATA_LOSS: 500, + grpc.StatusCode.UNIMPLEMENTED: 501, + grpc.StatusCode.UNAVAILABLE: 503, + grpc.StatusCode.DEADLINE_EXCEEDED: 504 +} + class OpenCensusServerInterceptor(grpc.ServerInterceptor): def __init__(self, sampler=None, exporter=None): @@ -58,6 +85,12 @@ def new_behavior(request_or_iterator, servicer_context): # invoke the original rpc behavior response_or_iterator = behavior(request_or_iterator, servicer_context) + + http_status_code = _convert_grpc_code_to_http_status_code( + servicer_context._state.code + ) + span.add_attribute(HTTP_STATUS_CODE, http_status_code) + if response_streaming: response_or_iterator = grpc_utils.wrap_iter_with_message_events( # noqa: E501 request_or_response_iter=response_or_iterator, @@ -109,28 +142,60 @@ def _start_server_span(self, servicer_context): ) span.span_kind = span_module.SpanKind.SERVER + + grpc_call_details = servicer_context._rpc_event.call_details + grpc_host = grpc_call_details.host.decode('utf-8') + grpc_method = grpc_call_details.method.decode('utf-8') + tracer.add_attribute_to_current_span( - attribute_key=attributes_helper.COMMON_ATTRIBUTES.get( - ATTRIBUTE_COMPONENT), - attribute_value='grpc') + COMPONENT, 'grpc' + ) + tracer.add_attribute_to_current_span( + GRPC_METHOD, grpc_method + ) + + tracer.add_attribute_to_current_span( + HTTP_HOST, grpc_host + ) + tracer.add_attribute_to_current_span( + HTTP_METHOD, 'POST' + ) + tracer.add_attribute_to_current_span( + HTTP_ROUTE, grpc_method + ) + tracer.add_attribute_to_current_span( + HTTP_PATH, grpc_method + ) + tracer.add_attribute_to_current_span( + HTTP_URL, 'grpc://' + grpc_host + grpc_method + ) execution_context.set_opencensus_tracer(tracer) execution_context.set_current_span(span) return span +def _convert_grpc_code_to_http_status_code(grpc_state_code): + """ + Converts a gRPC state code into the corresponding HTTP response status. + See: + https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto + """ + if grpc_state_code is None: + return 200 + else: + return GRPC_HTTP_STATUS_MAPPING.get(grpc_state_code, 500) + + def _add_exc_info(span): exc_type, exc_value, tb = sys.exc_info() - span.add_attribute( - attributes_helper.COMMON_ATTRIBUTES.get( - ATTRIBUTE_ERROR_MESSAGE), - str(exc_value) - ) + span.add_attribute(ERROR_MESSAGE, str(exc_value)) span.stack_trace = stack_trace.StackTrace.from_traceback(tb) span.status = status.Status( code=code_pb2.UNKNOWN, message=str(exc_value) ) + span.add_attribute(HTTP_STATUS_CODE, 500) def _wrap_rpc_behavior(handler, fn): diff --git a/contrib/opencensus-ext-grpc/opencensus/ext/grpc/utils.py b/contrib/opencensus-ext-grpc/opencensus/ext/grpc/utils.py index 222bceca8..7a36c3d84 100644 --- a/contrib/opencensus-ext-grpc/opencensus/ext/grpc/utils.py +++ b/contrib/opencensus-ext-grpc/opencensus/ext/grpc/utils.py @@ -2,8 +2,19 @@ from grpc.framework.foundation import future from grpc.framework.interfaces.face import face -from opencensus.trace import execution_context -from opencensus.trace import time_event + +from opencensus.trace import execution_context, time_event + + +def extract_byte_size(proto_message): + """Gets the byte size from a google.protobuf or proto-plus message""" + if hasattr(proto_message, "ByteSize"): + # google.protobuf message + return proto_message.ByteSize() + if hasattr(type(proto_message), "pb"): + # proto-plus message + return type(proto_message).pb(proto_message).ByteSize() + return None def add_message_event(proto_message, span, message_event_type, message_id=1): @@ -15,7 +26,7 @@ def add_message_event(proto_message, span, message_event_type, message_id=1): datetime.utcnow(), message_id, type=message_event_type, - uncompressed_size_bytes=proto_message.ByteSize() + uncompressed_size_bytes=extract_byte_size(proto_message), ) ) diff --git a/contrib/opencensus-ext-grpc/setup.py b/contrib/opencensus-ext-grpc/setup.py index b42e607a4..e6a6775d3 100644 --- a/contrib/opencensus-ext-grpc/setup.py +++ b/contrib/opencensus-ext-grpc/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,13 +34,15 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus gRPC Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ 'grpcio >= 1.0.0, < 2.0.0', - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-grpc/tests/test_client_interceptor.py b/contrib/opencensus-ext-grpc/tests/test_client_interceptor.py index e45096bb5..ab7851b20 100644 --- a/contrib/opencensus-ext-grpc/tests/test_client_interceptor.py +++ b/contrib/opencensus-ext-grpc/tests/test_client_interceptor.py @@ -13,14 +13,14 @@ # limitations under the License. import collections -import mock import threading import unittest +import grpc +import mock from google.api_core import bidi from google.protobuf import proto_builder from grpc.framework.foundation import logging_pool -import grpc from opencensus.ext.grpc import client_interceptor from opencensus.trace import execution_context diff --git a/contrib/opencensus-ext-grpc/tests/test_server_interceptor.py b/contrib/opencensus-ext-grpc/tests/test_server_interceptor.py index b5f3cad8d..d74f3003b 100644 --- a/contrib/opencensus-ext-grpc/tests/test_server_interceptor.py +++ b/contrib/opencensus-ext-grpc/tests/test_server_interceptor.py @@ -14,6 +14,7 @@ import unittest +import grpc import mock from google.rpc import code_pb2 @@ -22,6 +23,9 @@ from opencensus.trace import execution_context from opencensus.trace import span as span_module +MOCK_HOST = b'localhost:5000' +MOCK_METHOD = b'/helloworld.Greeter/SayHello' + class TestOpenCensusServerInterceptor(unittest.TestCase): def test_constructor(self): @@ -38,7 +42,10 @@ def test_intercept_service_no_metadata(self): '.tracer_module.Tracer', MockTracer) mock_context = mock.Mock() mock_context.invocation_metadata = mock.Mock(return_value=None) - mock_context._rpc_event.call_details.method = 'hello' + + mock_context._rpc_event.call_details.host = MOCK_HOST + mock_context._rpc_event.call_details.method = MOCK_METHOD + mock_context._state.code = grpc.StatusCode.OK interceptor = server_interceptor.OpenCensusServerInterceptor( None, None) mock_handler = mock.Mock() @@ -53,6 +60,13 @@ def test_intercept_service_no_metadata(self): expected_attributes = { 'component': 'grpc', + 'grpc.method': '/helloworld.Greeter/SayHello', + 'http.host': 'localhost:5000', + 'http.method': 'POST', + 'http.route': '/helloworld.Greeter/SayHello', + 'http.path': '/helloworld.Greeter/SayHello', + 'http.url': 'grpc://localhost:5000/helloworld.Greeter/SayHello', + 'http.status_code': 200 } self.assertEqual( @@ -78,7 +92,9 @@ def test_intercept_service(self): mock_handler.response_streaming = rsp_streaming mock_continuation = mock.Mock(return_value=mock_handler) - mock_context._rpc_event.call_details.method = 'hello' + mock_context._rpc_event.call_details.host = MOCK_HOST + mock_context._rpc_event.call_details.method = MOCK_METHOD + mock_context._state.code = grpc.StatusCode.OK interceptor = server_interceptor.OpenCensusServerInterceptor( None, None) @@ -89,6 +105,13 @@ def test_intercept_service(self): expected_attributes = { 'component': 'grpc', + 'grpc.method': '/helloworld.Greeter/SayHello', + 'http.host': 'localhost:5000', + 'http.method': 'POST', + 'http.route': '/helloworld.Greeter/SayHello', + 'http.path': '/helloworld.Greeter/SayHello', + 'http.url': 'grpc://localhost:5000/helloworld.Greeter/SayHello', # noqa: E501 + 'http.status_code': 200 } self.assertEqual( @@ -110,7 +133,9 @@ def test_intercept_handler_exception(self): None, None) mock_context = mock.Mock() mock_context.invocation_metadata = mock.Mock(return_value=None) - mock_context._rpc_event.call_details.method = 'hello' + + mock_context._rpc_event.call_details.host = MOCK_HOST + mock_context._rpc_event.call_details.method = MOCK_METHOD mock_handler = mock.Mock() mock_handler.request_streaming = req_streaming mock_handler.response_streaming = rsp_streaming @@ -128,6 +153,13 @@ def test_intercept_handler_exception(self): expected_attributes = { 'component': 'grpc', + 'grpc.method': '/helloworld.Greeter/SayHello', + 'http.host': 'localhost:5000', + 'http.method': 'POST', + 'http.route': '/helloworld.Greeter/SayHello', + 'http.path': '/helloworld.Greeter/SayHello', + 'http.url': 'grpc://localhost:5000/helloworld.Greeter/SayHello', # noqa: E501 + 'http.status_code': 500, 'error.message': 'Test' } @@ -147,8 +179,10 @@ def test_intercept_handler_exception(self): # check that the status obj is attached to the current span self.assertIsNotNone(current_span.status) - self.assertEqual(current_span.status.code, code_pb2.UNKNOWN) - self.assertEqual(current_span.status.message, 'Test') + self.assertEqual( + current_span.status.canonical_code, code_pb2.UNKNOWN + ) + self.assertEqual(current_span.status.description, 'Test') @mock.patch( 'opencensus.trace.execution_context.get_opencensus_tracer') @@ -165,6 +199,20 @@ def test__wrap_rpc_behavior_none(self): new_handler = server_interceptor._wrap_rpc_behavior(None, lambda: None) self.assertEqual(new_handler, None) + def test_extract_byte_size(self): + # should work with a google.protobuf message + google_protobuf_mock = mock.Mock() + google_protobuf_mock.ByteSize.return_value = 5 + self.assertEqual(grpc_utils.extract_byte_size(google_protobuf_mock), 5) + + # should work with a proto-plus style message + protoplus_protobuf_mock = mock.Mock(spec=[]) + type(protoplus_protobuf_mock).pb = mock.Mock() + type(protoplus_protobuf_mock).pb.return_value.ByteSize.return_value = 5 + self.assertEqual( + grpc_utils.extract_byte_size(protoplus_protobuf_mock), 5 + ) + class MockTracer(object): def __init__(self, *args, **kwargs): diff --git a/contrib/opencensus-ext-grpc/version.py b/contrib/opencensus-ext-grpc/version.py index deb2f374d..dffc606db 100644 --- a/contrib/opencensus-ext-grpc/version.py +++ b/contrib/opencensus-ext-grpc/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.4.dev0' +__version__ = '0.8.dev0' diff --git a/contrib/opencensus-ext-httplib/CHANGELOG.md b/contrib/opencensus-ext-httplib/CHANGELOG.md index 3f88b4499..d9ab3dee7 100644 --- a/contrib/opencensus-ext-httplib/CHANGELOG.md +++ b/contrib/opencensus-ext-httplib/CHANGELOG.md @@ -2,6 +2,28 @@ ## Unreleased +## 0.7.4 +Released 2021-01-14 + +- Change blacklist to excludelist +([#977](https://github.com/census-instrumentation/opencensus-python/pull/977)) + +## 0.7.3 +Released 2020-02-03 + +- Added `component` span attribute + +## 0.7.2 +Released 2019-08-26 + +- Updated `http.status_code` attribute to be an int. + ([#755](https://github.com/census-instrumentation/opencensus-python/pull/755)) + +## 0.7.1 +Released 2019-08-06 + + - Support exporter changes in `opencensus>=0.7.0` + ## 0.1.3 Released 2019-05-31 diff --git a/contrib/opencensus-ext-httplib/opencensus/ext/httplib/trace.py b/contrib/opencensus-ext-httplib/opencensus/ext/httplib/trace.py index ef49dace9..05a7bcb41 100644 --- a/contrib/opencensus-ext-httplib/opencensus/ext/httplib/trace.py +++ b/contrib/opencensus-ext-httplib/opencensus/ext/httplib/trace.py @@ -15,8 +15,7 @@ import logging import sys -from opencensus.trace import attributes_helper -from opencensus.trace import execution_context +from opencensus.trace import attributes_helper, execution_context, integrations from opencensus.trace import span as span_module from opencensus.trace import utils @@ -54,6 +53,9 @@ def trace_integration(tracer=None): wrapped_response = wrap_httplib_response(response_func) setattr(httplib.HTTPConnection, response_func.__name__, wrapped_response) + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.HTTP_LIB) + def wrap_httplib_request(request_func): """Wrap the httplib request function to trace. Create a new span and update @@ -61,17 +63,25 @@ def wrap_httplib_request(request_func): """ def call(self, method, url, body, headers, *args, **kwargs): + # Check if request was sent from an exporter. If so, do not wrap. + if execution_context.is_exporter(): + return request_func(self, method, url, body, + headers, *args, **kwargs) _tracer = execution_context.get_opencensus_tracer() - blacklist_hostnames = execution_context.get_opencensus_attr( - 'blacklist_hostnames') + excludelist_hostnames = execution_context.get_opencensus_attr( + 'excludelist_hostnames') dest_url = '{}:{}'.format(self.host, self.port) - if utils.disable_tracing_hostname(dest_url, blacklist_hostnames): + if utils.disable_tracing_hostname(dest_url, excludelist_hostnames): return request_func(self, method, url, body, headers, *args, **kwargs) _span = _tracer.start_span() _span.span_kind = span_module.SpanKind.CLIENT _span.name = '[httplib]{}'.format(request_func.__name__) + # Add the component type to attributes + _tracer.add_attribute_to_current_span( + "component", "HTTP") + # Add the request url to attributes _tracer.add_attribute_to_current_span(HTTP_URL, url) @@ -100,6 +110,9 @@ def wrap_httplib_response(response_func): """ def call(self, *args, **kwargs): + # Check if request was sent from an exporter. If so, do not wrap. + if execution_context.is_exporter(): + return response_func(self, *args, **kwargs) _tracer = execution_context.get_opencensus_tracer() current_span_id = execution_context.get_opencensus_attr( 'httplib/current_span_id') @@ -114,7 +127,7 @@ def call(self, *args, **kwargs): # Add the status code to attributes _tracer.add_attribute_to_current_span( - HTTP_STATUS_CODE, str(result.status)) + HTTP_STATUS_CODE, result.status) _tracer.end_span() return result diff --git a/contrib/opencensus-ext-httplib/setup.py b/contrib/opencensus-ext-httplib/setup.py index 7a29e9d24..9220435d1 100644 --- a/contrib/opencensus-ext-httplib/setup.py +++ b/contrib/opencensus-ext-httplib/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus httplib Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-httplib/tests/test_httplib_trace.py b/contrib/opencensus-ext-httplib/tests/test_httplib_trace.py index d2382eb87..cf0a30e94 100644 --- a/contrib/opencensus-ext-httplib/tests/test_httplib_trace.py +++ b/contrib/opencensus-ext-httplib/tests/test_httplib_trace.py @@ -75,6 +75,10 @@ def test_wrap_httplib_request(self): 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_tracer', return_value=mock_tracer) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) wrapped = trace.wrap_httplib_request(mock_request_func) @@ -84,10 +88,11 @@ def test_wrap_httplib_request(self): body = None headers = {} - with patch: + with patch, patch_thread: wrapped(mock_self, method, url, body, headers) - expected_attributes = {'http.url': url, 'http.method': method} + expected_attributes = {'component': 'HTTP', + 'http.url': url, 'http.method': method} expected_name = '[httplib]request' mock_request_func.assert_called_with(mock_self, method, url, body, { @@ -98,7 +103,7 @@ def test_wrap_httplib_request(self): self.assertEqual(span_module.SpanKind.CLIENT, mock_tracer.span.span_kind) - def test_wrap_httplib_request_blacklist_ok(self): + def test_wrap_httplib_request_excludelist_ok(self): mock_span = mock.Mock() span_id = '1234' mock_span.span_id = span_id @@ -114,6 +119,10 @@ def test_wrap_httplib_request_blacklist_ok(self): 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_attr', return_value=None) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) wrapped = trace.wrap_httplib_request(mock_request_func) @@ -123,14 +132,14 @@ def test_wrap_httplib_request_blacklist_ok(self): body = None headers = {} - with patch_tracer, patch_attr: + with patch_tracer, patch_attr, patch_thread: wrapped(mock_self, method, url, body, headers) mock_request_func.assert_called_with(mock_self, method, url, body, { 'traceparent': '00-123-456-01', }) - def test_wrap_httplib_request_blacklist_nok(self): + def test_wrap_httplib_request_excludelist_nok(self): mock_span = mock.Mock() span_id = '1234' mock_span.span_id = span_id @@ -146,6 +155,10 @@ def test_wrap_httplib_request_blacklist_nok(self): 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_attr', return_value=['localhost:8080']) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) wrapped = trace.wrap_httplib_request(mock_request_func) @@ -157,7 +170,30 @@ def test_wrap_httplib_request_blacklist_nok(self): body = None headers = {} - with patch_tracer, patch_attr: + with patch_tracer, patch_attr, patch_thread: + wrapped(mock_self, method, url, body, headers) + + mock_request_func.assert_called_with(mock_self, method, url, body, {}) + + def test_wrap_httplib_request_exporter_thread(self): + mock_request_func = mock.Mock() + mock_request_func.__name__ = 'request' + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=True) + + mock_self = mock.Mock() + mock_self.host = 'localhost' + mock_self.port = '8080' + method = 'GET' + url = 'http://{}:{}'.format(mock_self.host, mock_self.port) + body = None + headers = {} + + wrapped = trace.wrap_httplib_request(mock_request_func) + + with patch_thread: wrapped(mock_self, method, url, body, headers) mock_request_func.assert_called_with(mock_self, method, url, body, {}) @@ -181,10 +217,14 @@ def test_wrap_httplib_response(self): 'opencensus.ext.httplib.trace.' 'execution_context.get_opencensus_attr', return_value=span_id) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) wrapped = trace.wrap_httplib_response(mock_response_func) - with patch_tracer, patch_attr: + with patch_tracer, patch_attr, patch_thread: wrapped(mock.Mock()) expected_attributes = {'http.status_code': '200'} @@ -210,10 +250,14 @@ def test_wrap_httplib_response_no_open_span(self): 'opencensus.ext.httplib.trace.' 'execution_context.get_opencensus_attr', return_value='1111') + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) wrapped = trace.wrap_httplib_response(mock_response_func) - with patch_tracer, patch_attr: + with patch_tracer, patch_attr, patch_thread: wrapped(mock.Mock()) # Attribute should be empty as there is no matching span @@ -221,6 +265,39 @@ def test_wrap_httplib_response_no_open_span(self): self.assertEqual(expected_attributes, mock_tracer.span.attributes) + def test_wrap_httplib_response_exporter_thread(self): + mock_span = mock.Mock() + span_id = '1234' + mock_span.span_id = span_id + mock_span.attributes = {} + mock_tracer = MockTracer(mock_span) + mock_response_func = mock.Mock() + mock_result = mock.Mock() + mock_result.status = '200' + mock_response_func.return_value = mock_result + + patch_tracer = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'get_opencensus_tracer', + return_value=mock_tracer) + patch_attr = mock.patch( + 'opencensus.ext.httplib.trace.' + 'execution_context.get_opencensus_attr', + return_value='1111') + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=True) + + wrapped = trace.wrap_httplib_response(mock_response_func) + + with patch_tracer, patch_attr, patch_thread: + wrapped(mock.Mock()) + + expected_attributes = {} + + self.assertEqual(expected_attributes, mock_tracer.span.attributes) + class MockTracer(object): def __init__(self, span=None): diff --git a/contrib/opencensus-ext-httplib/version.py b/contrib/opencensus-ext-httplib/version.py index ff18aeb50..dffc606db 100644 --- a/contrib/opencensus-ext-httplib/version.py +++ b/contrib/opencensus-ext-httplib/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.2.dev0' +__version__ = '0.8.dev0' diff --git a/contrib/opencensus-ext-httpx/CHANGELOG.md b/contrib/opencensus-ext-httpx/CHANGELOG.md new file mode 100644 index 000000000..755e63048 --- /dev/null +++ b/contrib/opencensus-ext-httpx/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## Unreleased + +## 0.1.0 + +Released 2023-01-18 + +- Initial release +([#1098](https://github.com/census-instrumentation/opencensus-python/pull/1098)) diff --git a/contrib/opencensus-ext-httpx/README.rst b/contrib/opencensus-ext-httpx/README.rst new file mode 100644 index 000000000..6e4d5b87a --- /dev/null +++ b/contrib/opencensus-ext-httpx/README.rst @@ -0,0 +1,42 @@ +OpenCensus httpx Integration +============================================================================ + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opencensus-ext-httpx.svg + :target: https://pypi.org/project/opencensus-ext-httpx/ + +OpenCensus can trace HTTP requests made with the `httpx package `_. The request URL, +method, and status will be collected. + +You can enable httpx integration by specifying ``'httpx'`` to ``trace_integrations``. + +Only the hostname must be specified if only the hostname is specified in the URL request. + + +Installation +------------ + +:: + + pip install opencensus-ext-httpx + +Usage +----- + +.. code:: python + + import httpx + from opencensus.trace import config_integration + from opencensus.trace.tracer import Tracer + + if __name__ == '__main__': + config_integration.trace_integrations(['httpx']) + tracer = Tracer() + with tracer.span(name='parent'): + response = httpx.get(url='https://www.example.org') + +References +---------- + +* `OpenCensus Project `_ diff --git a/contrib/opencensus-ext-httpx/opencensus/__init__.py b/contrib/opencensus-ext-httpx/opencensus/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/opencensus-ext-httpx/opencensus/ext/__init__.py b/contrib/opencensus-ext-httpx/opencensus/ext/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/opencensus-ext-httpx/opencensus/ext/httpx/__init__.py b/contrib/opencensus-ext-httpx/opencensus/ext/httpx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/opencensus-ext-httpx/opencensus/ext/httpx/trace.py b/contrib/opencensus-ext-httpx/opencensus/ext/httpx/trace.py new file mode 100644 index 000000000..b77f29630 --- /dev/null +++ b/contrib/opencensus-ext-httpx/opencensus/ext/httpx/trace.py @@ -0,0 +1,118 @@ +import logging + +import httpx +import wrapt + +from opencensus.trace import ( + attributes_helper, + exceptions_status, + execution_context, + integrations, +) +from opencensus.trace import span as span_module +from opencensus.trace import utils + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +log = logging.getLogger(__name__) + +MODULE_NAME = "httpx" + +HTTP_HOST = attributes_helper.COMMON_ATTRIBUTES["HTTP_HOST"] +HTTP_METHOD = attributes_helper.COMMON_ATTRIBUTES["HTTP_METHOD"] +HTTP_PATH = attributes_helper.COMMON_ATTRIBUTES["HTTP_PATH"] +HTTP_ROUTE = attributes_helper.COMMON_ATTRIBUTES["HTTP_ROUTE"] +HTTP_STATUS_CODE = attributes_helper.COMMON_ATTRIBUTES["HTTP_STATUS_CODE"] +HTTP_URL = attributes_helper.COMMON_ATTRIBUTES["HTTP_URL"] + + +def trace_integration(tracer=None): + """Wrap the requests library to trace it.""" + log.info("Integrated module: {}".format(MODULE_NAME)) + + if tracer is not None: + # The execution_context tracer should never be None - if it has not + # been set it returns a no-op tracer. Most code in this library does + # not handle None being used in the execution context. + execution_context.set_opencensus_tracer(tracer) + + wrapt.wrap_function_wrapper( + MODULE_NAME, "Client.request", wrap_client_request + ) + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.HTTPX) + + +def wrap_client_request(wrapped, instance, args, kwargs): + """Wrap the session function to trace it.""" + # Check if request was sent from an exporter. If so, do not wrap. + if execution_context.is_exporter(): + return wrapped(*args, **kwargs) + + method = kwargs.get("method") or args[0] + url = kwargs.get("url") or args[1] + + excludelist_hostnames = execution_context.get_opencensus_attr( + "excludelist_hostnames" + ) + parsed_url = urlparse(url) + if parsed_url.port is None: + dest_url = parsed_url.hostname + else: + dest_url = "{}:{}".format(parsed_url.hostname, parsed_url.port) + if utils.disable_tracing_hostname(dest_url, excludelist_hostnames): + return wrapped(*args, **kwargs) + + path = parsed_url.path if parsed_url.path else "/" + + _tracer = execution_context.get_opencensus_tracer() + _span = _tracer.start_span() + + _span.name = "{}".format(path) + _span.span_kind = span_module.SpanKind.CLIENT + + try: + tracer_headers = _tracer.propagator.to_headers(_tracer.span_context) + kwargs.setdefault("headers", {}).update(tracer_headers) + except Exception: # pragma: NO COVER + pass + + # Add the component type to attributes + _tracer.add_attribute_to_current_span("component", "HTTP") + + # Add the requests host to attributes + _tracer.add_attribute_to_current_span(HTTP_HOST, dest_url) + + # Add the requests method to attributes + _tracer.add_attribute_to_current_span(HTTP_METHOD, method.upper()) + + # Add the requests path to attributes + _tracer.add_attribute_to_current_span(HTTP_PATH, path) + + # Add the requests url to attributes + _tracer.add_attribute_to_current_span(HTTP_URL, url) + + try: + result = wrapped(*args, **kwargs) + except httpx.TimeoutException: + _span.set_status(exceptions_status.TIMEOUT) + raise + except httpx.InvalidURL: + _span.set_status(exceptions_status.INVALID_URL) + raise + except Exception as e: + _span.set_status(exceptions_status.unknown(e)) + raise + else: + # Add the status code to attributes + _tracer.add_attribute_to_current_span( + HTTP_STATUS_CODE, result.status_code + ) + _span.set_status(utils.status_from_http_code(result.status_code)) + return result + finally: + _tracer.end_span() diff --git a/contrib/opencensus-ext-httpx/setup.cfg b/contrib/opencensus-ext-httpx/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/contrib/opencensus-ext-httpx/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/contrib/opencensus-ext-httpx/setup.py b/contrib/opencensus-ext-httpx/setup.py new file mode 100644 index 000000000..bf920415c --- /dev/null +++ b/contrib/opencensus-ext-httpx/setup.py @@ -0,0 +1,44 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from setuptools import find_packages, setup + +from version import __version__ + +setup( + name="opencensus-ext-httpx", + version=__version__, # noqa + author="Michał Klich", + author_email="michal@klichx.dev", + classifiers=[ + "Intended Audience :: Developers", + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + ], + description="OpenCensus HTTPX Integration", + include_package_data=True, + long_description=open('README.rst').read(), + install_requires=["opencensus >= 0.12.dev0, < 1.0.0", "httpx >= 0.22.0"], + extras_require={}, + license="Apache-2.0", + packages=find_packages(exclude=("tests",)), + namespace_packages=[], + url="", + zip_safe=False, +) diff --git a/contrib/opencensus-ext-httpx/tests/test_httpx_trace.py b/contrib/opencensus-ext-httpx/tests/test_httpx_trace.py new file mode 100644 index 000000000..ddb9cb87f --- /dev/null +++ b/contrib/opencensus-ext-httpx/tests/test_httpx_trace.py @@ -0,0 +1,474 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import httpx +import mock + +from opencensus.ext.httpx import trace +from opencensus.trace import execution_context +from opencensus.trace import span as span_module +from opencensus.trace import status as status_module +from opencensus.trace.tracers import noop_tracer + + +class Test_httpx_trace(unittest.TestCase): + def test_trace_integration(self): + mock_wrap = mock.Mock() + patch_wrapt = mock.patch("wrapt.wrap_function_wrapper", mock_wrap) + + with patch_wrapt: + trace.trace_integration() + + self.assertIsInstance( + execution_context.get_opencensus_tracer(), + noop_tracer.NoopTracer, + ) + mock_wrap.assert_called_once_with( + trace.MODULE_NAME, "Client.request", trace.wrap_client_request + ) + + def test_trace_integration_set_tracer(self): + mock_wrap = mock.Mock() + patch_wrapt = mock.patch("wrapt.wrap_function_wrapper", mock_wrap) + + class TmpTracer(noop_tracer.NoopTracer): + pass + + with patch_wrapt: + trace.trace_integration(tracer=TmpTracer()) + + self.assertIsInstance( + execution_context.get_opencensus_tracer(), TmpTracer + ) + mock_wrap.assert_called_once_with( + trace.MODULE_NAME, "Client.request", trace.wrap_client_request + ) + + def test_wrap_client_request(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {"x-trace": "some-value"}) + ) + + patch = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=False, + ) + + url = "http://localhost:8080/test" + request_method = "POST" + kwargs = {} + + with patch, patch_thread: + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), kwargs + ) + + expected_attributes = { + "component": "HTTP", + "http.host": "localhost:8080", + "http.method": "POST", + "http.path": "/test", + "http.status_code": 200, + "http.url": url, + } + expected_name = "/test" + expected_status = status_module.Status(0) + + self.assertEqual( + span_module.SpanKind.CLIENT, mock_tracer.current_span.span_kind + ) + self.assertEqual( + expected_attributes, mock_tracer.current_span.attributes + ) + self.assertEqual(kwargs["headers"]["x-trace"], "some-value") + self.assertEqual(expected_name, mock_tracer.current_span.name) + self.assertEqual( + expected_status.__dict__, mock_tracer.current_span.status.__dict__ + ) + + def test_wrap_client_request_excludelist_ok(self): + def wrapped(*args, **kwargs): + result = mock.Mock() + result.status_code = 200 + return result + + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {"x-trace": "some-value"}) + ) + + patch_tracer = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + patch_attr = mock.patch( + "opencensus.ext.httpx.trace.execution_context.get_opencensus_attr", + return_value=None, + ) + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=False, + ) + + url = "http://localhost/" + request_method = "POST" + + with patch_tracer, patch_attr, patch_thread: + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), {} + ) + + expected_name = "/" + self.assertEqual(expected_name, mock_tracer.current_span.name) + + def test_wrap_client_request_excludelist_nok(self): + def wrapped(*args, **kwargs): + result = mock.Mock() + result.status_code = 200 + return result + + mock_tracer = MockTracer() + + patch_tracer = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + patch_attr = mock.patch( + "opencensus.ext.httpx.trace.execution_context.get_opencensus_attr", + return_value=["localhost:8080"], + ) + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=False, + ) + + url = "http://localhost:8080" + request_method = "POST" + + with patch_tracer, patch_attr, patch_thread: + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), {} + ) + + self.assertEqual(None, mock_tracer.current_span) + + def test_wrap_client_request_exporter_thread(self): + def wrapped(*args, **kwargs): + result = mock.Mock() + result.status_code = 200 + return result + + mock_tracer = MockTracer() + + patch_tracer = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + patch_attr = mock.patch( + "opencensus.ext.httpx.trace.execution_context.get_opencensus_attr", + return_value=["localhost:8080"], + ) + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=True, + ) + + url = "http://localhost:8080" + request_method = "POST" + + with patch_tracer, patch_attr, patch_thread: + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), {} + ) + + self.assertEqual(None, mock_tracer.current_span) + + def test_header_is_passed_in(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {"x-trace": "some-value"}) + ) + + patch = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=False, + ) + + url = "http://localhost:8080" + request_method = "POST" + kwargs = {} + + with patch, patch_thread: + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), kwargs + ) + + self.assertEqual(kwargs["headers"]["x-trace"], "some-value") + + def test_headers_are_preserved(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {"x-trace": "some-value"}) + ) + + patch = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=False, + ) + + url = "http://localhost:8080" + request_method = "POST" + kwargs = {"headers": {"key": "value"}} + + with patch, patch_thread: + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), kwargs + ) + + self.assertEqual(kwargs["headers"]["key"], "value") + self.assertEqual(kwargs["headers"]["x-trace"], "some-value") + + def test_tracer_headers_are_overwritten(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {"x-trace": "some-value"}) + ) + + patch = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=False, + ) + + url = "http://localhost:8080" + request_method = "POST" + kwargs = {"headers": {"x-trace": "original-value"}} + + with patch, patch_thread: + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), kwargs + ) + + self.assertEqual(kwargs["headers"]["x-trace"], "some-value") + + def test_wrap_client_request_timeout(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + wrapped.side_effect = httpx.TimeoutException("timeout") + + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {"x-trace": "some-value"}) + ) + + patch = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=False, + ) + + url = "http://localhost:8080/test" + request_method = "POST" + kwargs = {} + + with patch, patch_thread: + with self.assertRaises(httpx.TimeoutException): + # breakpoint() + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), kwargs + ) + + expected_attributes = { + "component": "HTTP", + "http.host": "localhost:8080", + "http.method": "POST", + "http.path": "/test", + "http.url": url, + } + expected_name = "/test" + expected_status = status_module.Status(4, "request timed out") + + self.assertEqual( + span_module.SpanKind.CLIENT, mock_tracer.current_span.span_kind + ) + self.assertEqual( + expected_attributes, mock_tracer.current_span.attributes + ) + self.assertEqual(kwargs["headers"]["x-trace"], "some-value") + self.assertEqual(expected_name, mock_tracer.current_span.name) + self.assertEqual( + expected_status.__dict__, mock_tracer.current_span.status.__dict__ + ) + + def test_wrap_client_request_invalid_url(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + wrapped.side_effect = httpx.InvalidURL("invalid url") + + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {"x-trace": "some-value"}) + ) + + patch = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=False, + ) + + url = "http://localhost:8080/test" + request_method = "POST" + kwargs = {} + + with patch, patch_thread: + with self.assertRaises(httpx.InvalidURL): + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), kwargs + ) + + expected_attributes = { + "component": "HTTP", + "http.host": "localhost:8080", + "http.method": "POST", + "http.path": "/test", + "http.url": url, + } + expected_name = "/test" + expected_status = status_module.Status(3, "invalid URL") + + self.assertEqual( + span_module.SpanKind.CLIENT, mock_tracer.current_span.span_kind + ) + self.assertEqual( + expected_attributes, mock_tracer.current_span.attributes + ) + self.assertEqual(kwargs["headers"]["x-trace"], "some-value") + self.assertEqual(expected_name, mock_tracer.current_span.name) + self.assertEqual( + expected_status.__dict__, mock_tracer.current_span.status.__dict__ + ) + + def test_wrap_client_request_exception(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + wrapped.side_effect = httpx.TooManyRedirects("too many redirects") + + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {"x-trace": "some-value"}) + ) + + patch = mock.patch( + "opencensus.ext.httpx.trace.execution_context." + "get_opencensus_tracer", + return_value=mock_tracer, + ) + patch_thread = mock.patch( + "opencensus.ext.httpx.trace.execution_context.is_exporter", + return_value=False, + ) + + url = "http://localhost:8080/test" + request_method = "POST" + kwargs = {} + + with patch, patch_thread: + with self.assertRaises(httpx.TooManyRedirects): + trace.wrap_client_request( + wrapped, "Client.request", (request_method, url), kwargs + ) + + expected_attributes = { + "component": "HTTP", + "http.host": "localhost:8080", + "http.method": "POST", + "http.path": "/test", + "http.url": url, + } + expected_name = "/test" + expected_status = status_module.Status(2, "too many redirects") + + self.assertEqual( + span_module.SpanKind.CLIENT, mock_tracer.current_span.span_kind + ) + self.assertEqual( + expected_attributes, mock_tracer.current_span.attributes + ) + self.assertEqual(kwargs["headers"]["x-trace"], "some-value") + self.assertEqual(expected_name, mock_tracer.current_span.name) + self.assertEqual( + expected_status.__dict__, mock_tracer.current_span.status.__dict__ + ) + + +class MockTracer(object): + def __init__(self, propagator=None): + self.current_span = None + self.span_context = {} + self.propagator = propagator + + def start_span(self): + span = MockSpan() + self.current_span = span + return span + + def end_span(self): + pass + + def add_attribute_to_current_span(self, key, value): + self.current_span.attributes[key] = value + + +class MockSpan(object): + def __init__(self): + self.attributes = {} + + def set_status(self, status): + self.status = status diff --git a/contrib/opencensus-ext-httpx/version.py b/contrib/opencensus-ext-httpx/version.py new file mode 100644 index 000000000..506a49340 --- /dev/null +++ b/contrib/opencensus-ext-httpx/version.py @@ -0,0 +1 @@ +__version__ = '0.2.dev0' diff --git a/contrib/opencensus-ext-jaeger/CHANGELOG.md b/contrib/opencensus-ext-jaeger/CHANGELOG.md index b82a493db..bc2085635 100644 --- a/contrib/opencensus-ext-jaeger/CHANGELOG.md +++ b/contrib/opencensus-ext-jaeger/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +## 0.7.1 +Released 2019-08-05 + +- Update for core library changes + ## 0.2.2 Released 2019-05-31 diff --git a/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/__init__.py b/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/__init__.py index 509e054d2..e738573ed 100644 --- a/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/__init__.py +++ b/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/__init__.py @@ -184,12 +184,12 @@ def translate_to_jaeger(self, span_datas): tags.append(jaeger.Tag( key='status.code', vType=jaeger.TagType.LONG, - vLong=status.code)) + vLong=status.canonical_code)) tags.append(jaeger.Tag( key='status.message', vType=jaeger.TagType.STRING, - vStr=status.message)) + vStr=status.description)) refs = _extract_refs_from_span(span) logs = _extract_logs_from_span(span) diff --git a/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/gen/jaeger/agent.py b/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/gen/jaeger/agent.py index 176264a67..023a74b1d 100644 --- a/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/gen/jaeger/agent.py +++ b/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/gen/jaeger/agent.py @@ -6,9 +6,14 @@ # options string: py:new_style # -from thrift.Thrift import TType, TMessageType, TApplicationException -from thrift.Thrift import TProcessor +from thrift.Thrift import ( + TApplicationException, + TMessageType, + TProcessor, + TType, +) from thrift.transport import TTransport + from opencensus.ext.jaeger.trace_exporter.gen.jaeger import jaeger diff --git a/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/gen/jaeger/jaeger.py b/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/gen/jaeger/jaeger.py index 51c0ac617..e9fa4830b 100644 --- a/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/gen/jaeger/jaeger.py +++ b/contrib/opencensus-ext-jaeger/opencensus/ext/jaeger/trace_exporter/gen/jaeger/jaeger.py @@ -6,11 +6,16 @@ # options string: py:new_style # -from thrift.Thrift import TType, TMessageType, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -import sys import logging -from thrift.Thrift import TProcessor +import sys + +from thrift.protocol.TProtocol import TProtocolException +from thrift.Thrift import ( + TApplicationException, + TMessageType, + TProcessor, + TType, +) from thrift.transport import TTransport diff --git a/contrib/opencensus-ext-jaeger/setup.py b/contrib/opencensus-ext-jaeger/setup.py index a46f25a1b..18b918c7b 100644 --- a/contrib/opencensus-ext-jaeger/setup.py +++ b/contrib/opencensus-ext-jaeger/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Jaeger Trace Exporter', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', 'thrift >= 0.10.0', ], extras_require={}, diff --git a/contrib/opencensus-ext-jaeger/tests/test_jaeger_exporter.py b/contrib/opencensus-ext-jaeger/tests/test_jaeger_exporter.py index 67892058c..85c4b924f 100644 --- a/contrib/opencensus-ext-jaeger/tests/test_jaeger_exporter.py +++ b/contrib/opencensus-ext-jaeger/tests/test_jaeger_exporter.py @@ -18,8 +18,14 @@ from opencensus.ext.jaeger import trace_exporter from opencensus.ext.jaeger.trace_exporter.gen.jaeger import jaeger -from opencensus.trace import (attributes, link, span_context, span_data, - status, time_event) +from opencensus.trace import ( + attributes, + link, + span_context, + span_data, + status, + time_event, +) class TestJaegerExporter(unittest.TestCase): diff --git a/contrib/opencensus-ext-jaeger/version.py b/contrib/opencensus-ext-jaeger/version.py index bf7c8163b..dffc606db 100644 --- a/contrib/opencensus-ext-jaeger/version.py +++ b/contrib/opencensus-ext-jaeger/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.3.dev0' +__version__ = '0.8.dev0' diff --git a/contrib/opencensus-ext-logging/opencensus/ext/logging/trace.py b/contrib/opencensus-ext-logging/opencensus/ext/logging/trace.py index 851feaf09..13d95e846 100644 --- a/contrib/opencensus-ext-logging/opencensus/ext/logging/trace.py +++ b/contrib/opencensus-ext-logging/opencensus/ext/logging/trace.py @@ -15,6 +15,7 @@ import logging from opencensus.log import TraceLogger +from opencensus.trace import integrations def trace_integration(tracer=None): @@ -25,3 +26,5 @@ def trace_integration(tracer=None): context. """ logging.setLoggerClass(TraceLogger) + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.LOGGING) diff --git a/contrib/opencensus-ext-logging/setup.py b/contrib/opencensus-ext-logging/setup.py index e36b84dc3..af951dae2 100644 --- a/contrib/opencensus-ext-logging/setup.py +++ b/contrib/opencensus-ext-logging/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus logging Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-mysql/opencensus/ext/mysql/trace.py b/contrib/opencensus-ext-mysql/opencensus/ext/mysql/trace.py index a988a9214..5502195f6 100644 --- a/contrib/opencensus-ext-mysql/opencensus/ext/mysql/trace.py +++ b/contrib/opencensus-ext-mysql/opencensus/ext/mysql/trace.py @@ -14,9 +14,11 @@ import inspect import logging + import mysql.connector from opencensus.ext.dbapi import trace +from opencensus.trace import integrations MODULE_NAME = 'mysql' @@ -30,3 +32,5 @@ def trace_integration(tracer=None): conn_module = inspect.getmodule(conn_func) wrapped = trace.wrap_conn(conn_func) setattr(conn_module, CONN_WRAP_METHOD, wrapped) + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.MYSQL) diff --git a/contrib/opencensus-ext-mysql/setup.py b/contrib/opencensus-ext-mysql/setup.py index d84b765e4..bf0095165 100644 --- a/contrib/opencensus-ext-mysql/setup.py +++ b/contrib/opencensus-ext-mysql/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,13 +34,15 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus MySQL Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ 'mysql-connector >= 2.1.6, < 3.0.0', - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', 'opencensus-ext-dbapi >= 0.2.dev0, < 1.0.0', ], extras_require={}, diff --git a/contrib/opencensus-ext-ocagent/CHANGELOG.md b/contrib/opencensus-ext-ocagent/CHANGELOG.md index c1f0e931e..a1dcef595 100644 --- a/contrib/opencensus-ext-ocagent/CHANGELOG.md +++ b/contrib/opencensus-ext-ocagent/CHANGELOG.md @@ -2,12 +2,17 @@ ## Unreleased +## 0.7.1 +Released 2019-08-05 + +- Update for core library changes + ## 0.4.0 Released 2019-05-31 -- Remove well_known_types.Error and well_known_types.ParseError. -Note this could be a breaking change if you depend on an older -version of protobuf and use ParseError. +- Remove well_known_types.Error and well_known_types.ParseError. Note this + could be a breaking change if you depend on an older version of protobuf and + use ParseError. ## 0.3.0 Released 2019-04-24 diff --git a/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/stats_exporter/__init__.py b/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/stats_exporter/__init__.py index afb870c4d..3cca8b376 100644 --- a/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/stats_exporter/__init__.py +++ b/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/stats_exporter/__init__.py @@ -14,18 +14,20 @@ import logging +import grpc from google.api_core import bidi + from opencensus.common.monitored_resource import monitored_resource from opencensus.ext.ocagent import utils from opencensus.metrics import transport -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import value -from opencensus.proto.agent.metrics.v1 import metrics_service_pb2 -from opencensus.proto.agent.metrics.v1 import metrics_service_pb2_grpc +from opencensus.metrics.export import metric_descriptor, value +from opencensus.proto.agent.metrics.v1 import ( + metrics_service_pb2, + metrics_service_pb2_grpc, +) from opencensus.proto.metrics.v1 import metrics_pb2 from opencensus.proto.resource.v1 import resource_pb2 from opencensus.stats import stats -import grpc class StatsExporter(object): @@ -142,8 +144,8 @@ def _get_metric_descriptor_proto(descriptor): def _get_label_keys_proto(label_keys): return [ - metrics_pb2.LabelKey(key=l.key, description=l.description) - for l in label_keys + metrics_pb2.LabelKey(key=label.key, description=label.description) + for label in label_keys ] @@ -257,5 +259,5 @@ def new_stats_exporter(service_name, exporter = StatsExporter( ExportRpcHandler(_create_stub(endpoint), service_name, hostname)) - transport.get_exporter_thread(stats.stats, exporter, interval) + transport.get_exporter_thread([stats.stats], exporter, interval) return exporter diff --git a/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/trace_exporter/__init__.py b/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/trace_exporter/__init__.py index afb3f9bd3..ddd115755 100644 --- a/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/trace_exporter/__init__.py +++ b/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/trace_exporter/__init__.py @@ -14,13 +14,16 @@ """Export opencensus spans to ocagent""" from threading import Lock + import grpc from opencensus.common.transports import sync from opencensus.ext.ocagent import utils as ocagent_utils from opencensus.ext.ocagent.trace_exporter import utils -from opencensus.proto.agent.trace.v1 import trace_service_pb2 -from opencensus.proto.agent.trace.v1 import trace_service_pb2_grpc +from opencensus.proto.agent.trace.v1 import ( + trace_service_pb2, + trace_service_pb2_grpc, +) from opencensus.trace import base_exporter # Default agent endpoint diff --git a/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/trace_exporter/utils.py b/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/trace_exporter/utils.py index 27bfa0b5d..305347a3f 100644 --- a/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/trace_exporter/utils.py +++ b/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/trace_exporter/utils.py @@ -15,6 +15,7 @@ """Translates opencensus span data to trace proto""" from google.protobuf.wrappers_pb2 import BoolValue, UInt32Value + from opencensus.ext.ocagent import utils as ocagent_utils from opencensus.proto.trace.v1 import trace_pb2 @@ -43,8 +44,9 @@ def translate_to_trace_proto(span_data): span_data.start_time), end_time=ocagent_utils.proto_ts_from_datetime_str(span_data.end_time), status=trace_pb2.Status( - code=span_data.status.code, - message=span_data.status.message) + code=span_data.status.canonical_code, + message=span_data.status.description, + ) if span_data.status is not None else None, same_process_as_parent_span=BoolValue( value=span_data.same_process_as_parent_span) diff --git a/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/utils/__init__.py b/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/utils/__init__.py index 17347e3df..2196bc79b 100644 --- a/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/utils/__init__.py +++ b/contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/utils/__init__.py @@ -3,6 +3,7 @@ import socket from google.protobuf.timestamp_pb2 import Timestamp + from opencensus.common.version import __version__ as opencensus_version from opencensus.proto.agent.common.v1 import common_pb2 diff --git a/contrib/opencensus-ext-ocagent/setup.py b/contrib/opencensus-ext-ocagent/setup.py index 8c1b26858..ab82c086e 100644 --- a/contrib/opencensus-ext-ocagent/setup.py +++ b/contrib/opencensus-ext-ocagent/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,13 +34,15 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus OC-Agent Trace Exporter', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ 'grpcio >= 1.0.0, < 2.0.0', - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', 'opencensus-proto >= 0.1.0, < 1.0.0', ], extras_require={}, diff --git a/contrib/opencensus-ext-ocagent/tests/test_ocagent_utils.py b/contrib/opencensus-ext-ocagent/tests/test_ocagent_utils.py index 2ce995651..ec28ffdbf 100644 --- a/contrib/opencensus-ext-ocagent/tests/test_ocagent_utils.py +++ b/contrib/opencensus-ext-ocagent/tests/test_ocagent_utils.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime import unittest +from datetime import datetime from opencensus.common import utils as common_utils from opencensus.ext.ocagent import utils diff --git a/contrib/opencensus-ext-ocagent/tests/test_stats_exporter.py b/contrib/opencensus-ext-ocagent/tests/test_stats_exporter.py index 8eb91708f..796628531 100644 --- a/contrib/opencensus-ext-ocagent/tests/test_stats_exporter.py +++ b/contrib/opencensus-ext-ocagent/tests/test_stats_exporter.py @@ -12,30 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -from concurrent import futures -from datetime import datetime -import grpc -import mock import os import socket import threading import time import unittest +from concurrent import futures +from datetime import datetime +import grpc +import mock from google.protobuf import timestamp_pb2 -from opencensus.common import resource -from opencensus.common import utils + +from opencensus.common import resource, utils from opencensus.common.version import __version__ as opencensus_version from opencensus.ext.ocagent import stats_exporter as ocagent from opencensus.metrics import label_value -from opencensus.metrics.export import metric -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import point -from opencensus.metrics.export import time_series -from opencensus.metrics.export import value +from opencensus.metrics.export import ( + metric, + metric_descriptor, + point, + time_series, + value, +) from opencensus.proto.agent.common.v1 import common_pb2 -from opencensus.proto.agent.metrics.v1 import metrics_service_pb2 -from opencensus.proto.agent.metrics.v1 import metrics_service_pb2_grpc +from opencensus.proto.agent.metrics.v1 import ( + metrics_service_pb2, + metrics_service_pb2_grpc, +) from opencensus.proto.metrics.v1 import metrics_pb2 from opencensus.proto.resource.v1 import resource_pb2 from opencensus.stats import aggregation as aggregation_module @@ -342,7 +346,7 @@ def _helper(request_iterator, context): self._port, interval=0.1) - self.assertEqual(mock_transport.call_args[0][0], stats_module.stats) + self.assertEqual(mock_transport.call_args[0][0][0], stats_module.stats) self.assertEqual(mock_transport.call_args[0][1], exporter) self.assertEqual(mock_transport.call_args[0][2], 0.1) diff --git a/contrib/opencensus-ext-ocagent/tests/test_trace_exporter.py b/contrib/opencensus-ext-ocagent/tests/test_trace_exporter.py index fbd584dc8..077a1f5bf 100644 --- a/contrib/opencensus-ext-ocagent/tests/test_trace_exporter.py +++ b/contrib/opencensus-ext-ocagent/tests/test_trace_exporter.py @@ -13,19 +13,19 @@ # limitations under the License. import codecs -import grpc -import mock import os import socket import unittest +import grpc +import mock + from opencensus.common.version import __version__ from opencensus.ext.ocagent.trace_exporter import TraceExporter from opencensus.proto.trace.v1 import trace_config_pb2 from opencensus.trace import span_context as span_context_module from opencensus.trace import span_data as span_data_module - SERVICE_NAME = 'my-service' diff --git a/contrib/opencensus-ext-ocagent/tests/test_trace_exporter_utils.py b/contrib/opencensus-ext-ocagent/tests/test_trace_exporter_utils.py index 3c23e17c5..439114e1d 100644 --- a/contrib/opencensus-ext-ocagent/tests/test_trace_exporter_utils.py +++ b/contrib/opencensus-ext-ocagent/tests/test_trace_exporter_utils.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime -from datetime import timedelta import codecs import unittest +from datetime import datetime, timedelta from opencensus.ext.ocagent.trace_exporter import utils from opencensus.proto.trace.v1 import trace_pb2 diff --git a/contrib/opencensus-ext-ocagent/version.py b/contrib/opencensus-ext-ocagent/version.py index 235cf3f15..dffc606db 100644 --- a/contrib/opencensus-ext-ocagent/version.py +++ b/contrib/opencensus-ext-ocagent/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.5.dev0' +__version__ = '0.8.dev0' diff --git a/contrib/opencensus-ext-postgresql/opencensus/ext/postgresql/trace.py b/contrib/opencensus-ext-postgresql/opencensus/ext/postgresql/trace.py index 4c948af23..ac315273a 100644 --- a/contrib/opencensus-ext-postgresql/opencensus/ext/postgresql/trace.py +++ b/contrib/opencensus-ext-postgresql/opencensus/ext/postgresql/trace.py @@ -15,13 +15,13 @@ import inspect import logging -from opencensus.trace import execution_context -from opencensus.trace import span as span_module - import psycopg2 from psycopg2 import connect as pg_connect from psycopg2.extensions import cursor as pgcursor +from opencensus.trace import execution_context, integrations +from opencensus.trace import span as span_module + log = logging.getLogger(__name__) MODULE_NAME = 'postgresql' @@ -37,6 +37,8 @@ def trace_integration(tracer=None): conn_func = getattr(psycopg2, CONN_WRAP_METHOD) conn_module = inspect.getmodule(conn_func) setattr(conn_module, conn_func.__name__, connect) + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.POSTGRESQL) def connect(*args, **kwargs): diff --git a/contrib/opencensus-ext-postgresql/setup.py b/contrib/opencensus-ext-postgresql/setup.py index 5e8eb9bbe..60ef24bb3 100644 --- a/contrib/opencensus-ext-postgresql/setup.py +++ b/contrib/opencensus-ext-postgresql/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus PostgreSQL Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', 'psycopg2-binary >= 2.7.3.1', ], extras_require={}, diff --git a/contrib/opencensus-ext-prometheus/examples/prometheus.py b/contrib/opencensus-ext-prometheus/examples/prometheus.py index 5e808d007..99912f90e 100644 --- a/contrib/opencensus-ext-prometheus/examples/prometheus.py +++ b/contrib/opencensus-ext-prometheus/examples/prometheus.py @@ -16,6 +16,7 @@ import random import time +from pprint import pprint from opencensus.ext.prometheus import stats_exporter as prometheus from opencensus.stats import aggregation as aggregation_module @@ -25,7 +26,6 @@ from opencensus.tags import tag_key as tag_key_module from opencensus.tags import tag_map as tag_map_module from opencensus.tags import tag_value as tag_value_module -from pprint import pprint MiB = 1 << 20 FRONTEND_KEY = tag_key_module.TagKey("myorg_keys_frontend") diff --git a/contrib/opencensus-ext-prometheus/opencensus/ext/prometheus/stats_exporter/__init__.py b/contrib/opencensus-ext-prometheus/opencensus/ext/prometheus/stats_exporter/__init__.py index 82928850b..564ee3a11 100644 --- a/contrib/opencensus-ext-prometheus/opencensus/ext/prometheus/stats_exporter/__init__.py +++ b/contrib/opencensus-ext-prometheus/opencensus/ext/prometheus/stats_exporter/__init__.py @@ -12,20 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + from prometheus_client import start_http_server -from prometheus_client.core import CollectorRegistry -from prometheus_client.core import CounterMetricFamily -from prometheus_client.core import GaugeMetricFamily -from prometheus_client.core import HistogramMetricFamily -from prometheus_client.core import REGISTRY -from prometheus_client.core import UnknownMetricFamily +from prometheus_client.core import ( + REGISTRY, + CollectorRegistry, + CounterMetricFamily, + GaugeMetricFamily, + HistogramMetricFamily, + UnknownMetricFamily, +) from opencensus.common.transports import sync from opencensus.stats import aggregation_data as aggregation_data_module from opencensus.stats import base_exporter -import re - class Options(object): """ Options contains options for configuring the exporter. @@ -206,7 +208,7 @@ def to_metric(self, desc, tag_values, agg_data): return metric elif isinstance(agg_data, - aggregation_data_module.SumAggregationDataFloat): + aggregation_data_module.SumAggregationData): metric = UnknownMetricFamily(name=metric_name, documentation=metric_description, labels=label_keys) diff --git a/contrib/opencensus-ext-prometheus/setup.py b/contrib/opencensus-ext-prometheus/setup.py index a76eaddf3..3c25e9523 100644 --- a/contrib/opencensus-ext-prometheus/setup.py +++ b/contrib/opencensus-ext-prometheus/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Prometheus Exporter', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', 'prometheus_client >= 0.5.0, < 1.0.0', ], extras_require={}, diff --git a/contrib/opencensus-ext-prometheus/tests/test_prometheus_stats.py b/contrib/opencensus-ext-prometheus/tests/test_prometheus_stats.py index 5d9795534..bc19b47ef 100644 --- a/contrib/opencensus-ext-prometheus/tests/test_prometheus_stats.py +++ b/contrib/opencensus-ext-prometheus/tests/test_prometheus_stats.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime -import copy -import mock import unittest +from datetime import datetime +import mock from prometheus_client.core import Sample from opencensus.ext.prometheus import stats_exporter as prometheus @@ -140,7 +139,8 @@ def test_collector_to_metric_count(self): collector.register_view(view) desc = collector.registered_views[list(REGISTERED_VIEW)[0]] metric = collector.to_metric( - desc=desc, tag_values=[None], agg_data=agg.aggregation_data) + desc=desc, tag_values=[None], + agg_data=agg.new_aggregation_data(VIDEO_SIZE_MEASURE)) self.assertEqual(desc['name'], metric.name) self.assertEqual(desc['documentation'], metric.documentation) @@ -158,7 +158,8 @@ def test_collector_to_metric_sum(self): collector.register_view(view) desc = collector.registered_views[list(REGISTERED_VIEW)[0]] metric = collector.to_metric( - desc=desc, tag_values=[None], agg_data=agg.aggregation_data) + desc=desc, tag_values=[None], + agg_data=agg.new_aggregation_data(VIDEO_SIZE_MEASURE)) self.assertEqual(desc['name'], metric.name) self.assertEqual(desc['documentation'], metric.documentation) @@ -176,7 +177,8 @@ def test_collector_to_metric_last_value(self): collector.register_view(view) desc = collector.registered_views[list(REGISTERED_VIEW)[0]] metric = collector.to_metric( - desc=desc, tag_values=[None], agg_data=agg.aggregation_data) + desc=desc, tag_values=[None], + agg_data=agg.new_aggregation_data(VIDEO_SIZE_MEASURE)) self.assertEqual(desc['name'], metric.name) self.assertEqual(desc['documentation'], metric.documentation) @@ -189,7 +191,8 @@ def test_collector_to_metric_histogram(self): collector = prometheus.Collector(options=options) collector.register_view(VIDEO_SIZE_VIEW) desc = collector.registered_views[list(REGISTERED_VIEW)[0]] - distribution = copy.deepcopy(VIDEO_SIZE_DISTRIBUTION.aggregation_data) + distribution = VIDEO_SIZE_DISTRIBUTION.new_aggregation_data( + VIDEO_SIZE_MEASURE) distribution.add_sample(280.0 * MiB, None, None) metric = collector.to_metric( desc=desc, @@ -243,7 +246,7 @@ def test_collector_collect(self): metric = collector.to_metric( desc=desc, tag_values=[tag_value_module.TagValue("value")], - agg_data=agg.aggregation_data) + agg_data=agg.new_aggregation_data(VIDEO_SIZE_MEASURE)) self.assertEqual(desc['name'], metric.name) self.assertEqual(desc['documentation'], metric.documentation) @@ -262,7 +265,8 @@ def test_collector_collect_with_none_label_value(self): collector.register_view(view) desc = collector.registered_views['test3_new_view'] metric = collector.to_metric( - desc=desc, tag_values=[None], agg_data=agg.aggregation_data) + desc=desc, tag_values=[None], + agg_data=agg.new_aggregation_data(VIDEO_SIZE_MEASURE)) self.assertEqual(1, len(metric.samples)) sample = metric.samples[0] diff --git a/contrib/opencensus-ext-pymongo/CHANGELOG.md b/contrib/opencensus-ext-pymongo/CHANGELOG.md index 9ab12d5b5..6b3c87023 100644 --- a/contrib/opencensus-ext-pymongo/CHANGELOG.md +++ b/contrib/opencensus-ext-pymongo/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +## 0.7.1 +Released 2019-08-05 + +- Changed attributes names to make it compatible with + [OpenTelemetry](https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/data-semantic-conventions.md), + maintaining OpenCensus specs fidelity + ([#738](https://github.com/census-instrumentation/opencensus-python/pull/738)) + ## 0.1.3 Released 2019-05-31 diff --git a/contrib/opencensus-ext-pymongo/opencensus/ext/pymongo/trace.py b/contrib/opencensus-ext-pymongo/opencensus/ext/pymongo/trace.py index 655377013..cfc518b1b 100644 --- a/contrib/opencensus-ext-pymongo/opencensus/ext/pymongo/trace.py +++ b/contrib/opencensus-ext-pymongo/opencensus/ext/pymongo/trace.py @@ -14,11 +14,12 @@ import logging +from google.rpc import code_pb2 from pymongo import monitoring -from opencensus.trace import execution_context +from opencensus.trace import execution_context, integrations from opencensus.trace import span as span_module - +from opencensus.trace import status as status_module log = logging.getLogger(__name__) @@ -31,10 +32,11 @@ def trace_integration(tracer=None): """Integrate with pymongo to trace it using event listener.""" log.info('Integrated module: {}'.format(MODULE_NAME)) monitoring.register(MongoCommandListener(tracer=tracer)) + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.PYMONGO) class MongoCommandListener(monitoring.CommandListener): - def __init__(self, tracer=None): self._tracer = tracer @@ -44,30 +46,47 @@ def tracer(self): def started(self, event): span = self.tracer.start_span( - name='{}.{}.{}.{}'.format(MODULE_NAME, - event.database_name, - event.command.get(event.command_name), - event.command_name)) + name='{}.{}.{}.{}'.format( + MODULE_NAME, + event.database_name, + event.command.get(event.command_name), + event.command_name, + ) + ) span.span_kind = span_module.SpanKind.CLIENT + self.tracer.add_attribute_to_current_span('component', 'mongodb') + self.tracer.add_attribute_to_current_span('db.type', 'mongodb') + self.tracer.add_attribute_to_current_span( + 'db.instance', event.database_name + ) + self.tracer.add_attribute_to_current_span( + 'db.statement', event.command.get(event.command_name) + ) + for attr in COMMAND_ATTRIBUTES: _attr = event.command.get(attr) if _attr is not None: self.tracer.add_attribute_to_current_span(attr, str(_attr)) self.tracer.add_attribute_to_current_span( - 'request_id', event.request_id) + 'request_id', event.request_id + ) self.tracer.add_attribute_to_current_span( - 'connection_id', str(event.connection_id)) + 'connection_id', str(event.connection_id) + ) def succeeded(self, event): - self._stop('succeeded') + self._stop(code_pb2.OK) def failed(self, event): - self._stop('failed') - - def _stop(self, status): - self.tracer.add_attribute_to_current_span('status', status) - + self._stop(code_pb2.UNKNOWN, 'MongoDB error', event.failure) + + def _stop(self, code, message='', details=None): + span = self.tracer.current_span() + status = status_module.Status( + code=code, message=message, details=details + ) + span.set_status(status) self.tracer.end_span() diff --git a/contrib/opencensus-ext-pymongo/setup.py b/contrib/opencensus-ext-pymongo/setup.py index 0f26c789c..fd364eb32 100644 --- a/contrib/opencensus-ext-pymongo/setup.py +++ b/contrib/opencensus-ext-pymongo/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus pymongo Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', 'pymongo >= 3.1.0', ], extras_require={}, diff --git a/contrib/opencensus-ext-pymongo/tests/test_pymongo_trace.py b/contrib/opencensus-ext-pymongo/tests/test_pymongo_trace.py index 7ceffd2d0..a8e15250e 100644 --- a/contrib/opencensus-ext-pymongo/tests/test_pymongo_trace.py +++ b/contrib/opencensus-ext-pymongo/tests/test_pymongo_trace.py @@ -49,6 +49,10 @@ def test_started(self): } expected_attrs = { + 'component': 'mongodb', + 'db.type': 'mongodb', + 'db.instance': 'database_name', + 'db.statement': 'find', 'filter': 'filter', 'sort': 'sort', 'limit': 'limit', @@ -63,8 +67,8 @@ def test_started(self): trace.MongoCommandListener().started( event=MockEvent(command_attrs)) - self.assertEqual(mock_tracer.current_span.attributes, expected_attrs) - self.assertEqual(mock_tracer.current_span.name, expected_name) + self.assertEqual(mock_tracer.span.attributes, expected_attrs) + self.assertEqual(mock_tracer.span.name, expected_name) def test_succeed(self): mock_tracer = MockTracer() @@ -74,12 +78,16 @@ def test_succeed(self): 'opencensus.trace.execution_context.get_opencensus_tracer', return_value=mock_tracer) - expected_attrs = {'status': 'succeeded'} + expected_status = { + 'code': 0, + 'message': '', + 'details': None + } with patch: trace.MongoCommandListener().succeeded(event=MockEvent(None)) - self.assertEqual(mock_tracer.current_span.attributes, expected_attrs) + self.assertEqual(mock_tracer.span.status, expected_status) mock_tracer.end_span.assert_called_with() def test_failed(self): @@ -90,12 +98,16 @@ def test_failed(self): 'opencensus.trace.execution_context.get_opencensus_tracer', return_value=mock_tracer) - expected_attrs = {'status': 'failed'} + expected_status = { + 'code': 2, + 'message': 'MongoDB error', + 'details': 'failure' + } with patch: trace.MongoCommandListener().failed(event=MockEvent(None)) - self.assertEqual(mock_tracer.current_span.attributes, expected_attrs) + self.assertEqual(mock_tracer.span.status, expected_status) mock_tracer.end_span.assert_called_with() @@ -115,17 +127,31 @@ def __getattr__(self, item): return item +class MockSpan(object): + def __init__(self): + self.status = None + + def set_status(self, status): + self.status = { + 'code': status.canonical_code, + 'message': status.description, + 'details': status.details, + } + + class MockTracer(object): def __init__(self): - self.current_span = None + self.span = MockSpan() self.end_span = mock.Mock() def start_span(self, name=None): - span = mock.Mock() - span.name = name - span.attributes = {} - self.current_span = span - return span + self.span.name = name + self.span.attributes = {} + self.span.status = {} + return self.span def add_attribute_to_current_span(self, key, value): - self.current_span.attributes[key] = value + self.span.attributes[key] = value + + def current_span(self): + return self.span diff --git a/contrib/opencensus-ext-pymongo/version.py b/contrib/opencensus-ext-pymongo/version.py index ff18aeb50..dffc606db 100644 --- a/contrib/opencensus-ext-pymongo/version.py +++ b/contrib/opencensus-ext-pymongo/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.2.dev0' +__version__ = '0.8.dev0' diff --git a/contrib/opencensus-ext-pymysql/opencensus/ext/pymysql/trace.py b/contrib/opencensus-ext-pymysql/opencensus/ext/pymysql/trace.py index 8f52ce901..1821c51ed 100644 --- a/contrib/opencensus-ext-pymysql/opencensus/ext/pymysql/trace.py +++ b/contrib/opencensus-ext-pymysql/opencensus/ext/pymysql/trace.py @@ -14,9 +14,11 @@ import inspect import logging + import pymysql from opencensus.ext.dbapi import trace +from opencensus.trace import integrations MODULE_NAME = 'pymysql' @@ -30,3 +32,5 @@ def trace_integration(tracer=None): conn_module = inspect.getmodule(conn_func) wrapped = trace.wrap_conn(conn_func) setattr(conn_module, CONN_WRAP_METHOD, wrapped) + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.PYMYSQL) diff --git a/contrib/opencensus-ext-pymysql/setup.py b/contrib/opencensus-ext-pymysql/setup.py index 3f9d08d50..b1b398545 100644 --- a/contrib/opencensus-ext-pymysql/setup.py +++ b/contrib/opencensus-ext-pymysql/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,14 +34,16 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus PyMySQL Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ 'PyMySQL >= 0.7.11, < 1.0.0', - 'opencensus >= 0.7.dev0, < 1.0.0', - 'opencensus-ext-dbapi >= 0.2.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', + 'opencensus-ext-dbapi >= 0.1.2, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-pyramid/CHANGELOG.md b/contrib/opencensus-ext-pyramid/CHANGELOG.md index 46d3fbdcb..a342fe8b3 100644 --- a/contrib/opencensus-ext-pyramid/CHANGELOG.md +++ b/contrib/opencensus-ext-pyramid/CHANGELOG.md @@ -2,6 +2,24 @@ ## Unreleased +## 0.7.4 +Released 2021-01-14 + +- Change blacklist to excludelist +([#977](https://github.com/census-instrumentation/opencensus-python/pull/977)) + +## 0.7.1 +Released 2019-08-26 + +- Updated `http.status_code` attribute to be an int. + ([#755](https://github.com/census-instrumentation/opencensus-python/pull/755)) + +## 0.7.0 +Released 2019-07-31 + +- Updated span attributes to include some missing attributes listed [here](https://github.com/census-instrumentation/opencensus-specs/blob/master/trace/HTTP.md#attributes) +([#735](https://github.com/census-instrumentation/opencensus-python/pull/735)) + ## 0.3.0 Released 2019-04-24 diff --git a/contrib/opencensus-ext-pyramid/examples/app/__init__.py b/contrib/opencensus-ext-pyramid/examples/app/__init__.py index a4ca59a4b..7aae18fc5 100644 --- a/contrib/opencensus-ext-pyramid/examples/app/__init__.py +++ b/contrib/opencensus-ext-pyramid/examples/app/__init__.py @@ -13,7 +13,6 @@ # limitations under the License. import requests - from pyramid.config import Configurator from pyramid.response import Response from pyramid.tweens import MAIN @@ -28,7 +27,7 @@ def hello(request): @view_config(route_name='trace_requests') def trace_requests(request): response = requests.get('http://www.google.com') - return Response(str(response.status_code)) + return Response(response.status_code) def main(global_config, **settings): @@ -37,7 +36,7 @@ def main(global_config, **settings): config.add_route('hello', '/') config.add_route('trace_requests', '/requests') - config.add_tween('opencensus.trace.ext.pyramid' + config.add_tween('opencensus.ext.pyramid' '.pyramid_middleware.OpenCensusTweenFactory', over=MAIN) diff --git a/contrib/opencensus-ext-pyramid/examples/simple.py b/contrib/opencensus-ext-pyramid/examples/simple.py index 4de1a3487..001743831 100644 --- a/contrib/opencensus-ext-pyramid/examples/simple.py +++ b/contrib/opencensus-ext-pyramid/examples/simple.py @@ -14,12 +14,8 @@ from wsgiref.simple_server import make_server -from opencensus.trace import config_integration -from opencensus.trace import print_exporter -from opencensus.trace import samplers - from app import main - +from opencensus.trace import config_integration, print_exporter, samplers config_integration.trace_integrations(['requests']) diff --git a/contrib/opencensus-ext-pyramid/opencensus/ext/pyramid/config.py b/contrib/opencensus-ext-pyramid/opencensus/ext/pyramid/config.py index ec2267677..575432a2b 100644 --- a/contrib/opencensus-ext-pyramid/opencensus/ext/pyramid/config.py +++ b/contrib/opencensus-ext-pyramid/opencensus/ext/pyramid/config.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opencensus.trace import print_exporter -from opencensus.trace import samplers +from opencensus.trace import print_exporter, samplers from opencensus.trace.propagation import trace_context_http_header_format DEFAULT_PYRAMID_TRACER_CONFIG = { @@ -22,7 +21,7 @@ 'PROPAGATOR': trace_context_http_header_format.TraceContextPropagator(), # https://cloud.google.com/appengine/docs/flexible/python/ # how-instances-are-managed#health_checking - 'BLACKLIST_PATHS': ['_ah/health'], + 'EXCLUDELIST_PATHS': ['_ah/health'], } diff --git a/contrib/opencensus-ext-pyramid/opencensus/ext/pyramid/pyramid_middleware.py b/contrib/opencensus-ext-pyramid/opencensus/ext/pyramid/pyramid_middleware.py index a3cd0aeac..48b730201 100644 --- a/contrib/opencensus-ext-pyramid/opencensus/ext/pyramid/pyramid_middleware.py +++ b/contrib/opencensus-ext-pyramid/opencensus/ext/pyramid/pyramid_middleware.py @@ -15,17 +15,19 @@ import logging from opencensus.ext.pyramid.config import PyramidTraceSettings -from opencensus.trace import attributes_helper -from opencensus.trace import execution_context +from opencensus.trace import attributes_helper, execution_context, integrations from opencensus.trace import span as span_module from opencensus.trace import tracer as tracer_module from opencensus.trace import utils +HTTP_HOST = attributes_helper.COMMON_ATTRIBUTES['HTTP_HOST'] HTTP_METHOD = attributes_helper.COMMON_ATTRIBUTES['HTTP_METHOD'] +HTTP_PATH = attributes_helper.COMMON_ATTRIBUTES['HTTP_PATH'] +HTTP_ROUTE = attributes_helper.COMMON_ATTRIBUTES['HTTP_ROUTE'] HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL'] HTTP_STATUS_CODE = attributes_helper.COMMON_ATTRIBUTES['HTTP_STATUS_CODE'] -BLACKLIST_PATHS = 'BLACKLIST_PATHS' +EXCLUDELIST_PATHS = 'EXCLUDELIST_PATHS' log = logging.getLogger(__name__) @@ -60,7 +62,10 @@ def __init__(self, handler, registry): self.exporter = settings.EXPORTER self.propagator = settings.PROPAGATOR - self._blacklist_paths = settings.BLACKLIST_PATHS + self._excludelist_paths = settings.EXCLUDELIST_PATHS + + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.PYRAMID) def __call__(self, request): self._before_request(request) @@ -72,7 +77,7 @@ def __call__(self, request): return response def _before_request(self, request): - if utils.disable_tracing_url(request.path, self._blacklist_paths): + if utils.disable_tracing_url(request.path, self._excludelist_paths): return try: @@ -92,24 +97,33 @@ def _before_request(self, request): request.path) span.span_kind = span_module.SpanKind.SERVER + tracer.add_attribute_to_current_span( + attribute_key=HTTP_HOST, + attribute_value=request.host_url) tracer.add_attribute_to_current_span( attribute_key=HTTP_METHOD, attribute_value=request.method) tracer.add_attribute_to_current_span( - attribute_key=HTTP_URL, + attribute_key=HTTP_PATH, attribute_value=request.path) + tracer.add_attribute_to_current_span( + attribute_key=HTTP_ROUTE, + attribute_value=request.path) + tracer.add_attribute_to_current_span( + attribute_key=HTTP_URL, + attribute_value=request.url) except Exception: # pragma: NO COVER log.error('Failed to trace request', exc_info=True) def _after_request(self, request, response): - if utils.disable_tracing_url(request.path, self._blacklist_paths): + if utils.disable_tracing_url(request.path, self._excludelist_paths): return try: tracer = execution_context.get_opencensus_tracer() tracer.add_attribute_to_current_span( HTTP_STATUS_CODE, - str(response.status_code)) + response.status_code) tracer.end_span() tracer.finish() diff --git a/contrib/opencensus-ext-pyramid/setup.py b/contrib/opencensus-ext-pyramid/setup.py index dd9589a3c..49fe18476 100644 --- a/contrib/opencensus-ext-pyramid/setup.py +++ b/contrib/opencensus-ext-pyramid/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,13 +34,15 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Pyramid Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ 'pyramid >= 1.9.1, < 2.0.0', - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-pyramid/tests/test_pyramid_config.py b/contrib/opencensus-ext-pyramid/tests/test_pyramid_config.py index d8fc35f1d..4ff7addb0 100644 --- a/contrib/opencensus-ext-pyramid/tests/test_pyramid_config.py +++ b/contrib/opencensus-ext-pyramid/tests/test_pyramid_config.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock import unittest +import mock + from opencensus.ext.pyramid import config @@ -28,14 +29,14 @@ def test_trace_settings_default(self): assert trace_settings.SAMPLER == default_config['SAMPLER'] assert trace_settings.EXPORTER == default_config['EXPORTER'] assert trace_settings.PROPAGATOR == default_config['PROPAGATOR'] - assert trace_settings.BLACKLIST_PATHS == default_config[ - 'BLACKLIST_PATHS'] + assert trace_settings.EXCLUDELIST_PATHS == default_config[ + 'EXCLUDELIST_PATHS'] def test_trace_settings_override(self): mock_sampler = mock.Mock() mock_exporter = mock.Mock() mock_propagator = mock.Mock() - mock_blacklist_paths = ['foo/bar'] + mock_excludelist_paths = ['foo/bar'] registry = mock.Mock() registry.settings = { @@ -44,7 +45,7 @@ def test_trace_settings_override(self): 'SAMPLER': mock_sampler, 'EXPORTER': mock_exporter, 'PROPAGATOR': mock_propagator, - 'BLACKLIST_PATHS': mock_blacklist_paths, + 'EXCLUDELIST_PATHS': mock_excludelist_paths, }, }, } @@ -54,7 +55,7 @@ def test_trace_settings_override(self): assert trace_settings.SAMPLER == mock_sampler assert trace_settings.EXPORTER == mock_exporter assert trace_settings.PROPAGATOR == mock_propagator - assert trace_settings.BLACKLIST_PATHS == mock_blacklist_paths + assert trace_settings.EXCLUDELIST_PATHS == mock_excludelist_paths def test_trace_settings_invalid(self): registry = mock.Mock() diff --git a/contrib/opencensus-ext-pyramid/tests/test_pyramid_middleware.py b/contrib/opencensus-ext-pyramid/tests/test_pyramid_middleware.py index fb38a3e4c..63121b740 100644 --- a/contrib/opencensus-ext-pyramid/tests/test_pyramid_middleware.py +++ b/contrib/opencensus-ext-pyramid/tests/test_pyramid_middleware.py @@ -25,9 +25,7 @@ from opencensus.common.transports import sync from opencensus.ext.pyramid import pyramid_middleware from opencensus.ext.zipkin import trace_exporter as zipkin_exporter -from opencensus.trace import execution_context -from opencensus.trace import print_exporter -from opencensus.trace import samplers +from opencensus.trace import execution_context, print_exporter, samplers from opencensus.trace import span as span_module from opencensus.trace.blank_span import BlankSpan from opencensus.trace.propagation import trace_context_http_header_format @@ -155,8 +153,11 @@ def dummy_handler(request): span = tracer.current_span() expected_attributes = { - 'http.url': u'/', + 'http.host': u'http://example.com', 'http.method': 'GET', + 'http.path': u'/', + 'http.route': u'/', + 'http.url': u'http://example.com', } self.assertEqual(span.span_kind, span_module.SpanKind.SERVER) @@ -166,7 +167,7 @@ def dummy_handler(request): span_context = tracer.span_context self.assertEqual(span_context.trace_id, trace_id) - def test__before_request_blacklist(self): + def test__before_request_excludelist(self): pyramid_trace_header = 'traceparent' trace_id = '2dd43a1d6b2549c6bc2a1a54c2fc0b05' span_id = '6e0c63257de34c92' @@ -233,9 +234,12 @@ def dummy_handler(request): span = tracer.current_span() expected_attributes = { - 'http.url': u'/', + 'http.host': u'http://example.com', 'http.method': 'GET', - 'http.status_code': '200', + 'http.path': u'/', + 'http.route': u'/', + 'http.url': u'http://example.com', + 'http.status_code': 200, } self.assertEqual(span.parent_span.span_id, span_id) @@ -244,7 +248,7 @@ def dummy_handler(request): self.assertEqual(span.attributes, expected_attributes) - def test__after_request_blacklist(self): + def test__after_request_excludelist(self): pyramid_trace_header = 'traceparent' trace_id = '2dd43a1d6b2549c6bc2a1a54c2fc0b05' span_id = '6e0c63257de34c92' diff --git a/contrib/opencensus-ext-pyramid/version.py b/contrib/opencensus-ext-pyramid/version.py index deb2f374d..dffc606db 100644 --- a/contrib/opencensus-ext-pyramid/version.py +++ b/contrib/opencensus-ext-pyramid/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.4.dev0' +__version__ = '0.8.dev0' diff --git a/contrib/opencensus-ext-requests/CHANGELOG.md b/contrib/opencensus-ext-requests/CHANGELOG.md index f94e0621b..ac6f5f597 100644 --- a/contrib/opencensus-ext-requests/CHANGELOG.md +++ b/contrib/opencensus-ext-requests/CHANGELOG.md @@ -2,6 +2,46 @@ ## Unreleased +## 0.8.0 +Released 2022-08-03 + +- Move `version.py` file into `common` folder +([#1143](https://github.com/census-instrumentation/opencensus-python/pull/1143)) +- Add `requests` library as a hard dependency +([#1146](https://github.com/census-instrumentation/opencensus-python/pull/1146)) + +## 0.7.5 +Released 2021-05-13 + +- Fix duplicate spans being emitted from requests +([#1014](https://github.com/census-instrumentation/opencensus-python/pull/1014)) + +## 0.7.4 +Released 2021-01-14 + +- Change blacklist to excludelist +([#977](https://github.com/census-instrumentation/opencensus-python/pull/977)) + +## 0.7.3 +Released 2020-02-03 + +- Added `component` span attribute + +## 0.7.2 +Released 2019-08-26 + +- Added attributes following specs listed [here](https://github.com/census-instrumentation/opencensus-specs/blob/master/trace/HTTP.md#attributes) + ([#746](https://github.com/census-instrumentation/opencensus-python/pull/746)) +- Fixed span name + ([#746](https://github.com/census-instrumentation/opencensus-python/pull/746)) +- Fixed exception handling + ([#771](https://github.com/census-instrumentation/opencensus-python/pull/771)) + +## 0.7.1 +Released 2019-08-06 + + - Support exporter changes in `opencensus>=0.7.0` + ## 0.1.2 Released 2019-04-24 diff --git a/contrib/opencensus-ext-requests/README.rst b/contrib/opencensus-ext-requests/README.rst index 54762d7c0..dd0257d20 100644 --- a/contrib/opencensus-ext-requests/README.rst +++ b/contrib/opencensus-ext-requests/README.rst @@ -13,7 +13,7 @@ You can enable requests integration by specifying ``'requests'`` to ``trace_inte It's possible to configure a list of URL you don't want traced. By default the request to exporter won't be traced. It's configurable by giving an array of hostname/port to the attribute -``blacklist_hostnames`` in OpenCensus context's attributes: +``excludelist_hostnames`` in OpenCensus context's attributes: Only the hostname must be specified if only the hostname is specified in the URL request. diff --git a/contrib/opencensus-ext-requests/opencensus/ext/requests/common/__init__.py b/contrib/opencensus-ext-requests/opencensus/ext/requests/common/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/contrib/opencensus-ext-requests/opencensus/ext/requests/common/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/contrib/opencensus-ext-requests/opencensus/ext/requests/common/version.py b/contrib/opencensus-ext-requests/opencensus/ext/requests/common/version.py new file mode 100644 index 000000000..671fc3d04 --- /dev/null +++ b/contrib/opencensus-ext-requests/opencensus/ext/requests/common/version.py @@ -0,0 +1,15 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = '0.9.dev0' diff --git a/contrib/opencensus-ext-requests/opencensus/ext/requests/trace.py b/contrib/opencensus-ext-requests/opencensus/ext/requests/trace.py index 399bae24e..e1c7c1eef 100644 --- a/contrib/opencensus-ext-requests/opencensus/ext/requests/trace.py +++ b/contrib/opencensus-ext-requests/opencensus/ext/requests/trace.py @@ -13,28 +13,35 @@ # limitations under the License. import logging + import requests import wrapt + +from opencensus.trace import ( + attributes_helper, + exceptions_status, + execution_context, + integrations, +) +from opencensus.trace import span as span_module +from opencensus.trace import utils + try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse -from opencensus.trace import attributes_helper -from opencensus.trace import execution_context -from opencensus.trace import span as span_module -from opencensus.trace import utils log = logging.getLogger(__name__) MODULE_NAME = 'requests' -REQUESTS_WRAP_METHODS = ['get', 'post', 'put', 'delete', 'head', 'options'] -SESSION_WRAP_METHODS = 'request' -SESSION_CLASS_NAME = 'Session' - -HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL'] +HTTP_HOST = attributes_helper.COMMON_ATTRIBUTES['HTTP_HOST'] +HTTP_METHOD = attributes_helper.COMMON_ATTRIBUTES['HTTP_METHOD'] +HTTP_PATH = attributes_helper.COMMON_ATTRIBUTES['HTTP_PATH'] +HTTP_ROUTE = attributes_helper.COMMON_ATTRIBUTES['HTTP_ROUTE'] HTTP_STATUS_CODE = attributes_helper.COMMON_ATTRIBUTES['HTTP_STATUS_CODE'] +HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL'] def trace_integration(tracer=None): @@ -47,69 +54,44 @@ def trace_integration(tracer=None): # not handle None being used in the execution context. execution_context.set_opencensus_tracer(tracer) - # Wrap the requests functions - for func in REQUESTS_WRAP_METHODS: - requests_func = getattr(requests, func) - wrapped = wrap_requests(requests_func) - setattr(requests, requests_func.__name__, wrapped) - # Wrap Session class + # Since + # https://github.com/psf/requests/commit/d72d1162142d1bf8b1b5711c664fbbd674f349d1 + # (v0.7.0, Oct 23, 2011), get, post, etc are implemented via request which + # again, is implemented via Session.request (`Session` was named `session` + # before v1.0.0, Dec 17, 2012, see + # https://github.com/psf/requests/commit/4e5c4a6ab7bb0195dececdd19bb8505b872fe120) wrapt.wrap_function_wrapper( MODULE_NAME, 'Session.request', wrap_session_request) - - -def wrap_requests(requests_func): - """Wrap the requests function to trace it.""" - def call(url, *args, **kwargs): - blacklist_hostnames = execution_context.get_opencensus_attr( - 'blacklist_hostnames') - parsed_url = urlparse(url) - if parsed_url.port is None: - dest_url = parsed_url.hostname - else: - dest_url = '{}:{}'.format(parsed_url.hostname, parsed_url.port) - if utils.disable_tracing_hostname(dest_url, blacklist_hostnames): - return requests_func(url, *args, **kwargs) - - _tracer = execution_context.get_opencensus_tracer() - _span = _tracer.start_span() - _span.name = '[requests]{}'.format(requests_func.__name__) - _span.span_kind = span_module.SpanKind.CLIENT - - # Add the requests url to attributes - _tracer.add_attribute_to_current_span(HTTP_URL, url) - - result = requests_func(url, *args, **kwargs) - - # Add the status code to attributes - _tracer.add_attribute_to_current_span( - HTTP_STATUS_CODE, str(result.status_code)) - - _tracer.end_span() - return result - - return call + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.REQUESTS) def wrap_session_request(wrapped, instance, args, kwargs): """Wrap the session function to trace it.""" + # Check if request was sent from an exporter. If so, do not wrap. + if execution_context.is_exporter(): + return wrapped(*args, **kwargs) + method = kwargs.get('method') or args[0] url = kwargs.get('url') or args[1] - blacklist_hostnames = execution_context.get_opencensus_attr( - 'blacklist_hostnames') + excludelist_hostnames = execution_context.get_opencensus_attr( + 'excludelist_hostnames') parsed_url = urlparse(url) if parsed_url.port is None: dest_url = parsed_url.hostname else: dest_url = '{}:{}'.format(parsed_url.hostname, parsed_url.port) - if utils.disable_tracing_hostname(dest_url, blacklist_hostnames): + if utils.disable_tracing_hostname(dest_url, excludelist_hostnames): return wrapped(*args, **kwargs) + path = parsed_url.path if parsed_url.path else '/' + _tracer = execution_context.get_opencensus_tracer() _span = _tracer.start_span() - _span.name = '[requests]{}'.format(method) + _span.name = '{}'.format(path) _span.span_kind = span_module.SpanKind.CLIENT try: @@ -120,14 +102,44 @@ def wrap_session_request(wrapped, instance, args, kwargs): except Exception: # pragma: NO COVER pass - # Add the requests url to attributes - _tracer.add_attribute_to_current_span(HTTP_URL, url) + # Add the component type to attributes + _tracer.add_attribute_to_current_span( + "component", "HTTP") + + # Add the requests host to attributes + _tracer.add_attribute_to_current_span( + HTTP_HOST, dest_url) - result = wrapped(*args, **kwargs) + # Add the requests method to attributes + _tracer.add_attribute_to_current_span( + HTTP_METHOD, method.upper()) - # Add the status code to attributes + # Add the requests path to attributes _tracer.add_attribute_to_current_span( - HTTP_STATUS_CODE, str(result.status_code)) + HTTP_PATH, path) - _tracer.end_span() - return result + # Add the requests url to attributes + _tracer.add_attribute_to_current_span(HTTP_URL, url) + + try: + result = wrapped(*args, **kwargs) + except requests.Timeout: + _span.set_status(exceptions_status.TIMEOUT) + raise + except requests.URLRequired: + _span.set_status(exceptions_status.INVALID_URL) + raise + except Exception as e: + _span.set_status(exceptions_status.unknown(e)) + raise + else: + # Add the status code to attributes + _tracer.add_attribute_to_current_span( + HTTP_STATUS_CODE, result.status_code + ) + _span.set_status( + utils.status_from_http_code(result.status_code) + ) + return result + finally: + _tracer.end_span() diff --git a/contrib/opencensus-ext-requests/setup.py b/contrib/opencensus-ext-requests/setup.py index 761e324fd..5d6dad59f 100644 --- a/contrib/opencensus-ext-requests/setup.py +++ b/contrib/opencensus-ext-requests/setup.py @@ -12,13 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup -from version import __version__ +import os + +from setuptools import find_packages, setup + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, "opencensus", "ext", "requests", "common", "version.py" +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) setup( name='opencensus-ext-requests', - version=__version__, # noqa + version=PACKAGE_INFO["__version__"], # noqa author='OpenCensus Authors', author_email='census-developers@googlegroups.com', classifiers=[ @@ -34,12 +42,15 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Requests Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.12.dev0, < 1.0.0', + 'requests >= 2.19.0, < 3.0.0', 'wrapt >= 1.0.0, < 2.0.0', ], extras_require={}, diff --git a/contrib/opencensus-ext-requests/tests/test_requests_trace.py b/contrib/opencensus-ext-requests/tests/test_requests_trace.py index 01627ba13..37cd5a2f8 100644 --- a/contrib/opencensus-ext-requests/tests/test_requests_trace.py +++ b/contrib/opencensus-ext-requests/tests/test_requests_trace.py @@ -15,102 +15,136 @@ import unittest import mock -from opencensus.trace.tracers import noop_tracer +import requests from opencensus.ext.requests import trace -from opencensus.trace import span as span_module, execution_context +from opencensus.trace import execution_context +from opencensus.trace import span as span_module +from opencensus.trace import status as status_module +from opencensus.trace.tracers import noop_tracer class Test_requests_trace(unittest.TestCase): def test_trace_integration(self): mock_wrap = mock.Mock() - mock_requests = mock.Mock() - - wrap_result = 'wrap result' - mock_wrap.return_value = wrap_result - - for func in trace.REQUESTS_WRAP_METHODS: - mock_func = mock.Mock() - mock_func.__name__ = func - setattr(mock_requests, func, mock_func) - - patch_wrap = mock.patch( - 'opencensus.ext.requests.trace.wrap_requests', mock_wrap) - patch_requests = mock.patch( - 'opencensus.ext.requests.trace.requests', mock_requests) + patch_wrapt = mock.patch('wrapt.wrap_function_wrapper', mock_wrap) - with patch_wrap, patch_requests: + with patch_wrapt: trace.trace_integration() self.assertIsInstance(execution_context.get_opencensus_tracer(), noop_tracer.NoopTracer) - - for func in trace.REQUESTS_WRAP_METHODS: - self.assertEqual(getattr(mock_requests, func), wrap_result) + mock_wrap.assert_called_once_with( + trace.MODULE_NAME, + 'Session.request', + trace.wrap_session_request) def test_trace_integration_set_tracer(self): mock_wrap = mock.Mock() - mock_requests = mock.Mock() - - wrap_result = 'wrap result' - mock_wrap.return_value = wrap_result - - for func in trace.REQUESTS_WRAP_METHODS: - mock_func = mock.Mock() - mock_func.__name__ = func - setattr(mock_requests, func, mock_func) - - patch_wrap = mock.patch( - 'opencensus.ext.requests.trace.wrap_requests', mock_wrap) - patch_requests = mock.patch( - 'opencensus.ext.requests.trace.requests', mock_requests) + patch_wrapt = mock.patch('wrapt.wrap_function_wrapper', mock_wrap) class TmpTracer(noop_tracer.NoopTracer): pass - with patch_wrap, patch_requests: + with patch_wrapt: trace.trace_integration(tracer=TmpTracer()) self.assertIsInstance(execution_context.get_opencensus_tracer(), TmpTracer) + mock_wrap.assert_called_once_with( + trace.MODULE_NAME, + 'Session.request', + trace.wrap_session_request) - def test_wrap_requests(self): - mock_return = mock.Mock() - mock_return.status_code = 200 - return_value = mock_return - mock_func = mock.Mock() - mock_func.__name__ = 'get' - mock_func.return_value = return_value - mock_tracer = MockTracer() + def test_wrap_session_request(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {'x-trace': 'some-value'})) patch = mock.patch( 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_tracer', return_value=mock_tracer) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) - wrapped = trace.wrap_requests(mock_func) - - url = 'http://localhost:8080' - - with patch: - wrapped(url) + url = 'http://localhost:8080/test' + request_method = 'POST' + kwargs = {} - expected_attributes = {'http.url': url, 'http.status_code': '200'} - expected_name = '[requests]get' + with patch, patch_thread: + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), kwargs + ) + + expected_attributes = { + 'component': 'HTTP', + 'http.host': 'localhost:8080', + 'http.method': 'POST', + 'http.path': '/test', + 'http.status_code': 200, + 'http.url': url, + } + expected_name = '/test' + expected_status = status_module.Status(0) self.assertEqual(span_module.SpanKind.CLIENT, mock_tracer.current_span.span_kind) self.assertEqual(expected_attributes, mock_tracer.current_span.attributes) + self.assertEqual(kwargs['headers']['x-trace'], 'some-value') + self.assertEqual(expected_name, mock_tracer.current_span.name) + self.assertEqual( + expected_status.__dict__, + mock_tracer.current_span.status.__dict__ + ) + + def test_wrap_session_request_excludelist_ok(self): + def wrapped(*args, **kwargs): + result = mock.Mock() + result.status_code = 200 + return result + + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {'x-trace': 'some-value'})) + + patch_tracer = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'get_opencensus_tracer', + return_value=mock_tracer) + patch_attr = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'get_opencensus_attr', + return_value=None) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) + + url = 'http://localhost/' + request_method = 'POST' + + with patch_tracer, patch_attr, patch_thread: + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), {} + ) + + expected_name = '/' self.assertEqual(expected_name, mock_tracer.current_span.name) - def test_wrap_requests_blacklist_ok(self): - mock_return = mock.Mock() - mock_return.status_code = 200 - return_value = mock_return - mock_func = mock.Mock() - mock_func.__name__ = 'get' - mock_func.return_value = return_value + def test_wrap_session_request_excludelist_nok(self): + def wrapped(*args, **kwargs): + result = mock.Mock() + result.status_code = 200 + return result + mock_tracer = MockTracer() patch_tracer = mock.patch( @@ -121,25 +155,28 @@ def test_wrap_requests_blacklist_ok(self): 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_attr', return_value=['localhost:8080']) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) - wrapped = trace.wrap_requests(mock_func) - - url = 'http://localhost' + url = 'http://localhost:8080' + request_method = 'POST' - with patch_tracer, patch_attr: - wrapped(url) + with patch_tracer, patch_attr, patch_thread: + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), {} + ) - expected_name = '[requests]get' + self.assertEqual(None, mock_tracer.current_span) - self.assertEqual(expected_name, mock_tracer.current_span.name) + def test_wrap_session_request_exporter_thread(self): + def wrapped(*args, **kwargs): + result = mock.Mock() + result.status_code = 200 + return result - def test_wrap_requests_blacklist_nok(self): - mock_return = mock.Mock() - mock_return.status_code = 200 - return_value = mock_return - mock_func = mock.Mock() - mock_func.__name__ = 'get' - mock_func.return_value = return_value mock_tracer = MockTracer() patch_tracer = mock.patch( @@ -150,19 +187,24 @@ def test_wrap_requests_blacklist_nok(self): 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_attr', return_value=['localhost:8080']) - - wrapped = trace.wrap_requests(mock_func) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=True) url = 'http://localhost:8080' + request_method = 'POST' - with patch_tracer, patch_attr: - wrapped(url) + with patch_tracer, patch_attr, patch_thread: + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), {} + ) self.assertEqual(None, mock_tracer.current_span) - def test_wrap_session_request(self): + def test_header_is_passed_in(self): wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) - mock_tracer = MockTracer( propagator=mock.Mock( to_headers=lambda x: {'x-trace': 'some-value'})) @@ -171,81 +213,83 @@ def test_wrap_session_request(self): 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_tracer', return_value=mock_tracer) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) url = 'http://localhost:8080' request_method = 'POST' kwargs = {} - with patch: - trace.wrap_session_request(wrapped, 'Session.request', - (request_method, url), kwargs) - - expected_attributes = {'http.url': url, 'http.status_code': '200'} - expected_name = '[requests]POST' + with patch, patch_thread: + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), kwargs + ) - self.assertEqual(span_module.SpanKind.CLIENT, - mock_tracer.current_span.span_kind) - self.assertEqual(expected_attributes, - mock_tracer.current_span.attributes) self.assertEqual(kwargs['headers']['x-trace'], 'some-value') - self.assertEqual(expected_name, mock_tracer.current_span.name) - - def test_wrap_session_request_blacklist_ok(self): - def wrapped(*args, **kwargs): - result = mock.Mock() - result.status_code = 200 - return result + def test_headers_are_preserved(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) mock_tracer = MockTracer( propagator=mock.Mock( to_headers=lambda x: {'x-trace': 'some-value'})) - patch_tracer = mock.patch( + patch = mock.patch( 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_tracer', return_value=mock_tracer) - patch_attr = mock.patch( + patch_thread = mock.patch( 'opencensus.ext.requests.trace.execution_context.' - 'get_opencensus_attr', - return_value=None) + 'is_exporter', + return_value=False) - url = 'http://localhost' + url = 'http://localhost:8080' request_method = 'POST' + kwargs = {'headers': {'key': 'value'}} - with patch_tracer, patch_attr: - trace.wrap_session_request(wrapped, 'Session.request', - (request_method, url), {}) - - expected_name = '[requests]POST' - self.assertEqual(expected_name, mock_tracer.current_span.name) + with patch, patch_thread: + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), kwargs + ) - def test_wrap_session_request_blacklist_nok(self): - def wrapped(*args, **kwargs): - result = mock.Mock() - result.status_code = 200 - return result + self.assertEqual(kwargs['headers']['key'], 'value') + self.assertEqual(kwargs['headers']['x-trace'], 'some-value') - mock_tracer = MockTracer() + def test_tracer_headers_are_overwritten(self): + wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + mock_tracer = MockTracer( + propagator=mock.Mock( + to_headers=lambda x: {'x-trace': 'some-value'})) - patch_tracer = mock.patch( + patch = mock.patch( 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_tracer', return_value=mock_tracer) - patch_attr = mock.patch( + + patch_thread = mock.patch( 'opencensus.ext.requests.trace.execution_context.' - 'get_opencensus_attr', - return_value=['localhost:8080']) + 'is_exporter', + return_value=False) url = 'http://localhost:8080' request_method = 'POST' + kwargs = {'headers': {'x-trace': 'original-value'}} - with patch_tracer, patch_attr: - trace.wrap_session_request(wrapped, 'Session.request', - (request_method, url), {}) - self.assertEqual(None, mock_tracer.current_span) + with patch, patch_thread: + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), kwargs + ) - def test_header_is_passed_in(self): + self.assertEqual(kwargs['headers']['x-trace'], 'some-value') + + def test_wrap_session_request_timeout(self): wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + wrapped.side_effect = requests.Timeout + mock_tracer = MockTracer( propagator=mock.Mock( to_headers=lambda x: {'x-trace': 'some-value'})) @@ -254,19 +298,47 @@ def test_header_is_passed_in(self): 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_tracer', return_value=mock_tracer) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) - url = 'http://localhost:8080' + url = 'http://localhost:8080/test' request_method = 'POST' kwargs = {} - with patch: - trace.wrap_session_request(wrapped, 'Session.request', - (request_method, url), kwargs) + with patch, patch_thread: + with self.assertRaises(requests.Timeout): + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), kwargs + ) + + expected_attributes = { + 'component': 'HTTP', + 'http.host': 'localhost:8080', + 'http.method': 'POST', + 'http.path': '/test', + 'http.url': url, + } + expected_name = '/test' + expected_status = status_module.Status(4, 'request timed out') + self.assertEqual(span_module.SpanKind.CLIENT, + mock_tracer.current_span.span_kind) + self.assertEqual(expected_attributes, + mock_tracer.current_span.attributes) self.assertEqual(kwargs['headers']['x-trace'], 'some-value') + self.assertEqual(expected_name, mock_tracer.current_span.name) + self.assertEqual( + expected_status.__dict__, + mock_tracer.current_span.status.__dict__ + ) - def test_headers_are_preserved(self): + def test_wrap_session_request_invalid_url(self): wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + wrapped.side_effect = requests.URLRequired + mock_tracer = MockTracer( propagator=mock.Mock( to_headers=lambda x: {'x-trace': 'some-value'})) @@ -275,20 +347,47 @@ def test_headers_are_preserved(self): 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_tracer', return_value=mock_tracer) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) - url = 'http://localhost:8080' + url = 'http://localhost:8080/test' request_method = 'POST' - kwargs = {'headers': {'key': 'value'}} + kwargs = {} - with patch: - trace.wrap_session_request(wrapped, 'Session.request', - (request_method, url), kwargs) + with patch, patch_thread: + with self.assertRaises(requests.URLRequired): + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), kwargs + ) + + expected_attributes = { + 'component': 'HTTP', + 'http.host': 'localhost:8080', + 'http.method': 'POST', + 'http.path': '/test', + 'http.url': url, + } + expected_name = '/test' + expected_status = status_module.Status(3, 'invalid URL') - self.assertEqual(kwargs['headers']['key'], 'value') + self.assertEqual(span_module.SpanKind.CLIENT, + mock_tracer.current_span.span_kind) + self.assertEqual(expected_attributes, + mock_tracer.current_span.attributes) self.assertEqual(kwargs['headers']['x-trace'], 'some-value') + self.assertEqual(expected_name, mock_tracer.current_span.name) + self.assertEqual( + expected_status.__dict__, + mock_tracer.current_span.status.__dict__ + ) - def test_tracer_headers_are_overwritten(self): + def test_wrap_session_request_exception(self): wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) + wrapped.side_effect = requests.TooManyRedirects + mock_tracer = MockTracer( propagator=mock.Mock( to_headers=lambda x: {'x-trace': 'some-value'})) @@ -297,16 +396,42 @@ def test_tracer_headers_are_overwritten(self): 'opencensus.ext.requests.trace.execution_context.' 'get_opencensus_tracer', return_value=mock_tracer) + patch_thread = mock.patch( + 'opencensus.ext.requests.trace.execution_context.' + 'is_exporter', + return_value=False) - url = 'http://localhost:8080' + url = 'http://localhost:8080/test' request_method = 'POST' - kwargs = {'headers': {'x-trace': 'original-value'}} + kwargs = {} - with patch: - trace.wrap_session_request(wrapped, 'Session.request', - (request_method, url), kwargs) + with patch, patch_thread: + with self.assertRaises(requests.TooManyRedirects): + trace.wrap_session_request( + wrapped, 'Session.request', + (request_method, url), kwargs + ) + + expected_attributes = { + 'component': 'HTTP', + 'http.host': 'localhost:8080', + 'http.method': 'POST', + 'http.path': '/test', + 'http.url': url, + } + expected_name = '/test' + expected_status = status_module.Status(2, '') + self.assertEqual(span_module.SpanKind.CLIENT, + mock_tracer.current_span.span_kind) + self.assertEqual(expected_attributes, + mock_tracer.current_span.attributes) self.assertEqual(kwargs['headers']['x-trace'], 'some-value') + self.assertEqual(expected_name, mock_tracer.current_span.name) + self.assertEqual( + expected_status.__dict__, + mock_tracer.current_span.status.__dict__ + ) class MockTracer(object): @@ -316,8 +441,7 @@ def __init__(self, propagator=None): self.propagator = propagator def start_span(self): - span = mock.Mock() - span.attributes = {} + span = MockSpan() self.current_span = span return span @@ -326,3 +450,11 @@ def end_span(self): def add_attribute_to_current_span(self, key, value): self.current_span.attributes[key] = value + + +class MockSpan(object): + def __init__(self): + self.attributes = {} + + def set_status(self, status): + self.status = status diff --git a/contrib/opencensus-ext-sqlalchemy/opencensus/ext/sqlalchemy/trace.py b/contrib/opencensus-ext-sqlalchemy/opencensus/ext/sqlalchemy/trace.py index 100170fe1..ea0660bab 100644 --- a/contrib/opencensus-ext-sqlalchemy/opencensus/ext/sqlalchemy/trace.py +++ b/contrib/opencensus-ext-sqlalchemy/opencensus/ext/sqlalchemy/trace.py @@ -14,11 +14,9 @@ import logging -from sqlalchemy import engine -from sqlalchemy import event +from sqlalchemy import engine, event - -from opencensus.trace import execution_context +from opencensus.trace import execution_context, integrations from opencensus.trace import span as span_module log = logging.getLogger(__name__) @@ -33,6 +31,8 @@ def trace_integration(tracer=None): """ log.info('Integrated module: {}'.format(MODULE_NAME)) trace_engine(engine.Engine) + # pylint: disable=protected-access + integrations.add_integration(integrations._Integrations.SQLALCHEMY) def trace_engine(engine): diff --git a/contrib/opencensus-ext-sqlalchemy/setup.py b/contrib/opencensus-ext-sqlalchemy/setup.py index e89bd146b..f13621cbd 100644 --- a/contrib/opencensus-ext-sqlalchemy/setup.py +++ b/contrib/opencensus-ext-sqlalchemy/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,13 +34,15 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus SQLAlchemy Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', - 'SQLAlchemy >= 1.1.14, < 2.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', + 'SQLAlchemy >= 1.1.14', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-sqlalchemy/tests/test_sqlalchemy_trace.py b/contrib/opencensus-ext-sqlalchemy/tests/test_sqlalchemy_trace.py index 3f8f9ba06..9df47e415 100644 --- a/contrib/opencensus-ext-sqlalchemy/tests/test_sqlalchemy_trace.py +++ b/contrib/opencensus-ext-sqlalchemy/tests/test_sqlalchemy_trace.py @@ -16,8 +16,8 @@ import mock -from opencensus.trace import span as span_module from opencensus.ext.sqlalchemy import trace +from opencensus.trace import span as span_module class Test_sqlalchemy_trace(unittest.TestCase): diff --git a/contrib/opencensus-ext-stackdriver/CHANGELOG.md b/contrib/opencensus-ext-stackdriver/CHANGELOG.md index ae8f2670e..a24b98d23 100644 --- a/contrib/opencensus-ext-stackdriver/CHANGELOG.md +++ b/contrib/opencensus-ext-stackdriver/CHANGELOG.md @@ -2,6 +2,39 @@ ## Unreleased +## 0.8.0 +Released 2021-08-16 + +- Upgrade google-cloud-monitoring dep to v2.X, update usage + [#1060](https://github.com/census-instrumentation/opencensus-python/pull/1060) +- Drop python <3.6 in opencensus-ext-stackdriver + [#1056](https://github.com/census-instrumentation/opencensus-python/pull/1056) + +## 0.7.4 +Released 2020-10-14 + +- Change default transporter in stackdriver exporter +([#929](https://github.com/census-instrumentation/opencensus-python/pull/929)) + +## 0.7.3 +Released 2020-06-29 + +- Add mean property for distribution values +([#919](https://github.com/census-instrumentation/opencensus-python/pull/919)) + +## 0.7.2 +Released 2019-08-26 + +- Delete SD integ test metric descriptors +([#770](https://github.com/census-instrumentation/opencensus-python/pull/770)) +- Updated `http.status_code` attribute to be an int. +([#755](https://github.com/census-instrumentation/opencensus-python/pull/755)) + +## 0.7.1 +Released 2019-08-05 + +- Support exporter changes in `opencensus>=0.7.0` + ## 0.4.0 Released 2019-05-31 diff --git a/contrib/opencensus-ext-stackdriver/README.rst b/contrib/opencensus-ext-stackdriver/README.rst index 46b878a15..3dd36e177 100644 --- a/contrib/opencensus-ext-stackdriver/README.rst +++ b/contrib/opencensus-ext-stackdriver/README.rst @@ -32,23 +32,22 @@ This example shows how to report the traces to Stackdriver Trace: :: - pip install google-cloud-trace - pipenv install google-cloud-trace + pip install google-cloud-trace<1.0.0 + pipenv install google-cloud-trace<1.0.0 -By default, traces are exported synchronously, which introduces latency during -your code's execution. To avoid blocking code execution, you can initialize -your exporter to use a background thread. +By default, traces are exported asynchronously, to reduce latency during +your code's execution. If you would like to export data on the main thread +use the synchronous transporter: -This example shows how to configure OpenCensus to use a background thread: .. code:: python - from opencensus.common.transports.async_ import AsyncTransport + from opencensus.common.transports.sync import SyncTransport from opencensus.ext.stackdriver import trace_exporter as stackdriver_exporter from opencensus.trace import tracer as tracer_module exporter = stackdriver_exporter.StackdriverExporter( - project_id='your_cloud_project', transport=AsyncTransport) + project_id='your_cloud_project', transport=SyncTransport) tracer = tracer_module.Tracer(exporter=exporter) Stats diff --git a/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/stats_exporter/__init__.py b/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/stats_exporter/__init__.py index 637202d85..8b8cb1098 100644 --- a/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/stats_exporter/__init__.py +++ b/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/stats_exporter/__init__.py @@ -12,32 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime import itertools import os import platform import re import string import threading +from datetime import datetime +import google.auth +from google.api import metric_pb2 from google.api_core.gapic_v1 import client_info from google.cloud import monitoring_v3 -import google.auth +from google.protobuf import timestamp_pb2 from opencensus.common import utils -from opencensus.common.monitored_resource import aws_identity_doc_utils -from opencensus.common.monitored_resource import gcp_metadata_config -from opencensus.common.monitored_resource import k8s_utils -from opencensus.common.monitored_resource import monitored_resource +from opencensus.common.monitored_resource import ( + aws_identity_doc_utils, + gcp_metadata_config, + k8s_utils, + monitored_resource, +) from opencensus.common.version import __version__ -from opencensus.metrics import label_key -from opencensus.metrics import label_value -from opencensus.metrics import transport +from opencensus.metrics import label_key, label_value, transport from opencensus.metrics.export import metric as metric_module from opencensus.metrics.export import metric_descriptor from opencensus.stats import stats - MAX_TIME_SERIES_PER_UPLOAD = 200 OPENCENSUS_TASK = "opencensus_task" OPENCENSUS_TASK_DESCRIPTION = "Opencensus task identifier" @@ -52,20 +53,20 @@ # OC metric descriptor type to SD metric kind and value type OC_MD_TO_SD_TYPE = { metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64: - (monitoring_v3.enums.MetricDescriptor.MetricKind.CUMULATIVE, - monitoring_v3.enums.MetricDescriptor.ValueType.INT64), + (metric_pb2.MetricDescriptor.MetricKind.CUMULATIVE, + metric_pb2.MetricDescriptor.ValueType.INT64), metric_descriptor.MetricDescriptorType.CUMULATIVE_DOUBLE: - (monitoring_v3.enums.MetricDescriptor.MetricKind.CUMULATIVE, - monitoring_v3.enums.MetricDescriptor.ValueType.DOUBLE), + (metric_pb2.MetricDescriptor.MetricKind.CUMULATIVE, + metric_pb2.MetricDescriptor.ValueType.DOUBLE), metric_descriptor.MetricDescriptorType.CUMULATIVE_DISTRIBUTION: - (monitoring_v3.enums.MetricDescriptor.MetricKind.CUMULATIVE, - monitoring_v3.enums.MetricDescriptor.ValueType.DISTRIBUTION), + (metric_pb2.MetricDescriptor.MetricKind.CUMULATIVE, + metric_pb2.MetricDescriptor.ValueType.DISTRIBUTION), metric_descriptor.MetricDescriptorType.GAUGE_INT64: - (monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE, - monitoring_v3.enums.MetricDescriptor.ValueType.INT64), + (metric_pb2.MetricDescriptor.MetricKind.GAUGE, + metric_pb2.MetricDescriptor.ValueType.INT64), metric_descriptor.MetricDescriptorType.GAUGE_DOUBLE: - (monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE, - monitoring_v3.enums.MetricDescriptor.ValueType.DOUBLE) + (metric_pb2.MetricDescriptor.MetricKind.GAUGE, + metric_pb2.MetricDescriptor.ValueType.DOUBLE) } @@ -159,7 +160,9 @@ def export_metrics(self, metrics): ts_batches = self.create_batched_time_series(metrics) for ts_batch in ts_batches: self.client.create_time_series( - self.client.project_path(self.options.project_id), ts_batch) + name=self.client.common_project_path(self.options.project_id), + time_series=ts_batch, + ) def create_batched_time_series(self, metrics, batch_size=MAX_TIME_SERIES_PER_UPLOAD): @@ -174,7 +177,7 @@ def create_time_series_list(self, metric): def _convert_series(self, metric, ts): """Convert an OC timeseries to a SD series.""" - series = monitoring_v3.types.TimeSeries() + series = monitoring_v3.TimeSeries() series.metric.type = self.get_metric_type(metric.descriptor) for lk, lv in self.options.default_monitoring_labels.items(): @@ -188,9 +191,10 @@ def _convert_series(self, metric, ts): set_monitored_resource(series, self.options.resource) for point in ts.points: - sd_point = series.points.add() + sd_point = monitoring_v3.Point() # this just modifies points, no return self._convert_point(metric, ts, point, sd_point) + series.points.append(sd_point) return series def _convert_point(self, metric, ts, point, sd_point): @@ -202,6 +206,7 @@ def _convert_point(self, metric, ts, point, sd_point): sd_dist_val.count = point.value.count sd_dist_val.sum_of_squared_deviation =\ point.value.sum_of_squared_deviation + sd_dist_val.mean = point.value.sum / sd_dist_val.count assert sd_dist_val.bucket_options.explicit_buckets.bounds == [] sd_dist_val.bucket_options.explicit_buckets.bounds.extend( @@ -245,14 +250,17 @@ def _convert_point(self, metric, ts, point, sd_point): timestamp_start = (start - EPOCH_DATETIME).total_seconds() timestamp_end = (end - EPOCH_DATETIME).total_seconds() - sd_point.interval.end_time.seconds = int(timestamp_end) - - secs = sd_point.interval.end_time.seconds - sd_point.interval.end_time.nanos = int((timestamp_end - secs) * 1e9) + end_time_pb = timestamp_pb2.Timestamp() + end_time_pb.seconds = int(timestamp_end) + end_time_pb.nanos = int((timestamp_end - end_time_pb.seconds) * 1e9) + sd_point.interval.end_time = end_time_pb - start_time = sd_point.interval.start_time - start_time.seconds = int(timestamp_start) - start_time.nanos = int((timestamp_start - start_time.seconds) * 1e9) + start_time_pb = timestamp_pb2.Timestamp() + start_time_pb.seconds = int(timestamp_start) + start_time_pb.nanos = int( + (timestamp_start - start_time_pb.seconds) * 1e9, + ) + sd_point.interval.start_time = start_time_pb def get_metric_type(self, oc_md): """Get a SD metric type for an OC metric descriptor.""" @@ -273,7 +281,7 @@ def get_metric_descriptor(self, oc_md): desc_labels = new_label_descriptors( self.options.default_monitoring_labels, oc_md.label_keys) - descriptor = monitoring_v3.types.MetricDescriptor(labels=desc_labels) + descriptor = metric_pb2.MetricDescriptor(labels=desc_labels) metric_type = self.get_metric_type(oc_md) descriptor.type = metric_type descriptor.metric_kind = metric_kind @@ -295,8 +303,10 @@ def register_metric_descriptor(self, oc_md): return self._md_cache[metric_type] descriptor = self.get_metric_descriptor(oc_md) - project_name = self.client.project_path(self.options.project_id) - sd_md = self.client.create_metric_descriptor(project_name, descriptor) + project_name = self.client.common_project_path(self.options.project_id) + sd_md = self.client.create_metric_descriptor( + name=project_name, metric_descriptor=descriptor + ) with self._md_lock: self._md_cache[metric_type] = sd_md return sd_md @@ -401,7 +411,7 @@ def new_stats_exporter(options=None, interval=None): client = monitoring_v3.MetricServiceClient(client_info=ci) exporter = StackdriverStatsExporter(client=client, options=options) - transport.get_exporter_thread(stats.stats, exporter, interval=interval) + transport.get_exporter_thread([stats.stats], exporter, interval=interval) return exporter diff --git a/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/trace_exporter/__init__.py b/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/trace_exporter/__init__.py index d83d63ba4..b09e69647 100644 --- a/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/trace_exporter/__init__.py +++ b/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/trace_exporter/__init__.py @@ -12,20 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict import os +from collections import defaultdict from google.cloud.trace.client import Client -from opencensus.common.monitored_resource import aws_identity_doc_utils -from opencensus.common.monitored_resource import gcp_metadata_config -from opencensus.common.monitored_resource import k8s_utils -from opencensus.common.monitored_resource import monitored_resource -from opencensus.common.transports import sync +from opencensus.common.monitored_resource import ( + aws_identity_doc_utils, + gcp_metadata_config, + k8s_utils, + monitored_resource, +) +from opencensus.common.transports.async_ import AsyncTransport from opencensus.common.version import __version__ -from opencensus.trace import attributes_helper -from opencensus.trace import base_exporter -from opencensus.trace import span_data +from opencensus.trace import attributes_helper, base_exporter, span_data from opencensus.trace.attributes import Attributes # Agent @@ -175,12 +175,12 @@ class StackdriverExporter(base_exporter.Exporter): :param transport: Class for creating new transport objects. It should extend from the base_exporter :class:`.Transport` type and implement :meth:`.Transport.export`. Defaults to - :class:`.SyncTransport`. The other option is - :class:`.AsyncTransport`. + :class:`.AsyncTransport`. The other option is + :class:`.SyncTransport`. """ def __init__(self, client=None, project_id=None, - transport=sync.SyncTransport): + transport=AsyncTransport): # The client will handle the case when project_id is None if client is None: client = Client(project=project_id) @@ -274,6 +274,16 @@ def map_attributes(self, attribute_map): if (attribute_key in ATTRIBUTE_MAPPING): new_key = ATTRIBUTE_MAPPING.get(attribute_key) value[new_key] = value.pop(attribute_key) + if new_key == '/http/status_code': + # workaround: Stackdriver expects status to be str + hack = value[new_key] + hack = hack['int_value'] + if not isinstance(hack, int): + hack = hack['value'] + value[new_key] = {'string_value': { + 'truncated_byte_count': 0, + 'value': str(hack), + }} return attribute_map diff --git a/contrib/opencensus-ext-stackdriver/setup.py b/contrib/opencensus-ext-stackdriver/setup.py index 06abc5bc4..8d62651cb 100644 --- a/contrib/opencensus-ext-stackdriver/setup.py +++ b/contrib/opencensus-ext-stackdriver/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -27,22 +27,22 @@ 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Stackdriver Trace Exporter', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'google-cloud-monitoring >= 0.30.0, < 1.0.0', + 'google-cloud-monitoring ~= 2.0', 'google-cloud-trace >= 0.20.0, < 1.0.0', - 'opencensus >= 0.7.dev0, < 1.0.0', + 'rsa <= 4.0; python_version<="3.4"', + 'opencensus >= 0.9.dev0, < 1.0.0', ], + python_requires=">=3.6", extras_require={}, license='Apache-2.0', packages=find_packages(exclude=('examples', 'tests',)), diff --git a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_exporter.py b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_exporter.py index c64806825..c48e21681 100644 --- a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_exporter.py +++ b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_exporter.py @@ -448,8 +448,9 @@ def test_translate_common_attributes_to_stackdriver(self): } }, '/http/status_code': { - 'int_value': { - 'value': 200 + 'string_value': { + 'truncated_byte_count': 0, + 'value': '200' } }, '/http/url': { @@ -532,6 +533,37 @@ def test_translate_common_attributes_to_stackdriver(self): exporter.map_attributes(attributes) self.assertEqual(attributes, expected_attributes) + def test_translate_common_attributes_status_code(self): + project_id = 'PROJECT' + client = mock.Mock() + client.project = project_id + exporter = trace_exporter.StackdriverExporter( + client=client, project_id=project_id) + + attributes = { + 'outer key': 'some value', + 'attributeMap': { + 'http.status_code': { + 'int_value': 200 + } + } + } + + expected_attributes = { + 'outer key': 'some value', + 'attributeMap': { + '/http/status_code': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': '200' + } + } + } + } + + exporter.map_attributes(attributes) + self.assertEqual(attributes, expected_attributes) + class Test_set_attributes_gae(unittest.TestCase): @mock.patch('opencensus.ext.stackdriver.trace_exporter.' diff --git a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py index f57532acf..9c2f0e183 100644 --- a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py +++ b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py @@ -12,24 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime -import mock import unittest +from datetime import datetime, timedelta -from google.cloud import monitoring_v3 import google.auth +import mock +from google.api import metric_pb2 +from google.cloud import monitoring_v3 from opencensus.common import utils from opencensus.common.version import __version__ from opencensus.ext.stackdriver import stats_exporter as stackdriver -from opencensus.metrics import label_key -from opencensus.metrics import label_value +from opencensus.metrics import label_key, label_value from opencensus.metrics import transport as transport_module -from opencensus.metrics.export import metric -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import point -from opencensus.metrics.export import time_series -from opencensus.metrics.export import value +from opencensus.metrics.export import ( + metric, + metric_descriptor, + point, + time_series, + value, +) from opencensus.stats import aggregation as aggregation_module from opencensus.stats import aggregation_data as aggregation_data_module from opencensus.stats import execution_context @@ -432,12 +434,12 @@ def test_get_metric_descriptor(self): sd_md = exporter.get_metric_descriptor(oc_md) self.assertEqual( sd_md.metric_kind, - monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE) + metric_pb2.MetricDescriptor.MetricKind.GAUGE) self.assertEqual( sd_md.value_type, - monitoring_v3.enums.MetricDescriptor.ValueType.INT64) + metric_pb2.MetricDescriptor.ValueType.INT64) - self.assertIsInstance(sd_md, monitoring_v3.types.MetricDescriptor) + self.assertIsInstance(sd_md, metric_pb2.MetricDescriptor) exporter.client.create_metric_descriptor.assert_not_called() def test_get_metric_descriptor_bad_type(self): @@ -529,19 +531,30 @@ def test_export_metrics(self): exporter.export_metrics([mm]) self.assertEqual(exporter.client.create_time_series.call_count, 1) - sd_args = exporter.client.create_time_series.call_args[0][1] + sd_args = exporter.client.create_time_series.call_args.kwargs[ + 'time_series' + ] self.assertEqual(len(sd_args), 1) - [sd_arg] = exporter.client.create_time_series.call_args[0][1] + [sd_arg] = exporter.client.create_time_series.call_args.kwargs[ + 'time_series' + ] self.assertEqual(sd_arg.points[0].value.int64_value, 123) -class MockMetricExporterTask(object): - """Testing mock of metrics.transport.MetricExporterTask. +class MockPeriodicMetricTask(object): + """Testing mock of metrics.transport.PeriodicMetricTask. Simulate calling export asynchronously from another thread synchronously from this one. """ - def __init__(self, interval=None, function=None, args=None, kwargs=None): + def __init__( + self, + interval=None, + function=None, + args=None, + kwargs=None, + name=None + ): self.function = function self.logger = mock.Mock() self.start = mock.Mock() @@ -560,7 +573,7 @@ def step(self): class MockGetExporterThread(object): """Intercept calls to get_exporter_thread. - To get a reference to the running MetricExporterTask created by + To get a reference to the running PeriodicMetricTask created by get_exporter_thread. """ def __init__(self): @@ -594,8 +607,8 @@ class TestAsyncStatsExport(unittest.TestCase): def setUp(self): patcher = mock.patch( - 'opencensus.metrics.transport.MetricExporterTask', - MockMetricExporterTask) + 'opencensus.metrics.transport.PeriodicMetricTask', + MockPeriodicMetricTask) patcher.start() self.addCleanup(patcher.stop) @@ -646,21 +659,25 @@ def test_export_single_metric(self, mock_stats, mock_client): exporter.client.create_metric_descriptor.call_count, 1) md_call_arg =\ - exporter.client.create_metric_descriptor.call_args[0][1] + exporter.client.create_metric_descriptor.call_args.kwargs[ + 'metric_descriptor' + ] self.assertEqual( md_call_arg.metric_kind, - monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE + metric_pb2.MetricDescriptor.MetricKind.GAUGE ) self.assertEqual( md_call_arg.value_type, - monitoring_v3.enums.MetricDescriptor.ValueType.INT64 + metric_pb2.MetricDescriptor.ValueType.INT64 ) exporter.client.create_time_series.assert_called() self.assertEqual( exporter.client.create_time_series.call_count, 1) - ts_call_arg = exporter.client.create_time_series.call_args[0][1] + ts_call_arg = exporter.client.create_time_series.call_args.kwargs[ + 'time_series' + ] self.assertEqual(len(ts_call_arg), 1) self.assertEqual(len(ts_call_arg[0].points), 1) self.assertEqual(ts_call_arg[0].points[0].value.int64_value, 123) @@ -804,6 +821,9 @@ def test_create_timeseries(self, monitor_resource_mock): v_data = measure_map.measure_to_view_map.get_view( VIDEO_SIZE_VIEW_NAME, None) + v_data._start_time = ( + TEST_TIME - timedelta(minutes=1) + ).strftime(stackdriver.EPOCH_PATTERN) v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) time_series_list = exporter.create_time_series_list(v_data) @@ -821,20 +841,25 @@ def test_create_timeseries(self, monitor_resource_mock): self.assertEqual(len(time_series.points), 1) value = time_series.points[0].value - self.assertEqual(value.distribution_value.count, 1) - time_series_list = exporter.create_time_series_list(v_data) - - self.assertEqual(len(time_series_list), 1) - time_series = time_series_list[0] - self.check_labels( - time_series.metric.labels, {FRONTEND_KEY_CLEAN: "1200"}, - include_opencensus=True) - self.assertIsNotNone(time_series.resource) + expected_distb = google.api.distribution_pb2.Distribution( + count=1, + mean=26214400.0, + bucket_options=google.api.distribution_pb2.Distribution.BucketOptions( # noqa + explicit_buckets=google.api.distribution_pb2.Distribution.BucketOptions.Explicit( # noqa + bounds=[0.0, 16777216.0, 268435456.0])), + bucket_counts=[0, 0, 1, 0] + ) + self.assertEqual(value.distribution_value, expected_distb) - self.assertEqual(len(time_series.points), 1) - value = time_series.points[0].value - self.assertEqual(value.distribution_value.count, 1) + start_time_pb = ( + time_series.points[0].interval.start_time.timestamp_pb() + ) + end_time_pb = time_series.points[0].interval.end_time.timestamp_pb() + self.assertEqual(start_time_pb.seconds, 1545699663) + self.assertEqual(start_time_pb.nanos, 4053) + self.assertEqual(end_time_pb.seconds, 1545699723) + self.assertEqual(end_time_pb.nanos, 4053) @mock.patch('opencensus.ext.stackdriver.stats_exporter.' 'monitored_resource.get_instance') @@ -1020,7 +1045,7 @@ def test_create_timeseries_str_tagvalue(self, monitor_resource_mock): self.assertIsNotNone(time_series.resource) self.assertEqual(len(time_series.points), 1) - expected_value = monitoring_v3.types.TypedValue() + expected_value = monitoring_v3.TypedValue() # TODO: #565 expected_value.double_value = 25.0 * MiB self.assertEqual(time_series.points[0].value, expected_value) @@ -1062,7 +1087,7 @@ def test_create_timeseries_str_tagvalue_count_aggregtation( self.assertIsNotNone(time_series.resource) self.assertEqual(len(time_series.points), 1) - expected_value = monitoring_v3.types.TypedValue() + expected_value = monitoring_v3.TypedValue() expected_value.int64_value = 3 self.assertEqual(time_series.points[0].value, expected_value) @@ -1103,7 +1128,7 @@ def test_create_timeseries_last_value_float_tagvalue( self.assertIsNotNone(time_series.resource) self.assertEqual(len(time_series.points), 1) - expected_value = monitoring_v3.types.TypedValue() + expected_value = monitoring_v3.TypedValue() expected_value.double_value = 25.7 * MiB self.assertEqual(time_series.points[0].value, expected_value) @@ -1159,7 +1184,7 @@ def test_create_timeseries_float_tagvalue(self, monitor_resource_mock): self.assertIsNotNone(time_series.resource) self.assertEqual(len(time_series.points), 1) - expected_value = monitoring_v3.types.TypedValue() + expected_value = monitoring_v3.TypedValue() expected_value.double_value = 2.2 + 25 * MiB self.assertEqual(time_series.points[0].value, expected_value) @@ -1212,7 +1237,16 @@ def test_create_timeseries_multiple_tag_values(self, self.assertEqual(len(ts1.points), 1) value1 = ts1.points[0].value - self.assertEqual(value1.distribution_value.count, 1) + + expected_distb = google.api.distribution_pb2.Distribution( + count=1, + mean=26214400.0, + bucket_options=google.api.distribution_pb2.Distribution.BucketOptions( # noqa + explicit_buckets=google.api.distribution_pb2.Distribution.BucketOptions.Explicit( # noqa + bounds=[0.0, 16777216.0, 268435456.0])), + bucket_counts=[0, 0, 1, 0] + ) + self.assertEqual(value1.distribution_value, expected_distb) # Verify second time series self.assertEqual(ts2.resource.type, "global") @@ -1223,7 +1257,16 @@ def test_create_timeseries_multiple_tag_values(self, self.assertEqual(len(ts2.points), 1) value2 = ts2.points[0].value - self.assertEqual(value2.distribution_value.count, 1) + + expected_distb = google.api.distribution_pb2.Distribution( + count=1, + mean=12582912.0, + bucket_options=google.api.distribution_pb2.Distribution.BucketOptions( # noqa + explicit_buckets=google.api.distribution_pb2.Distribution.BucketOptions.Explicit( # noqa + bounds=[0.0, 16777216.0, 268435456.0])), + bucket_counts=[0, 1, 0, 0] + ) + self.assertEqual(value2.distribution_value, expected_distb) @mock.patch('opencensus.ext.stackdriver.stats_exporter.' 'monitored_resource.get_instance', @@ -1268,15 +1311,14 @@ def test_create_timeseries_disjoint_tags(self, monitoring_resoure_mock): self.assertIsNotNone(time_series.resource) self.assertEqual(len(time_series.points), 1) - expected_value = monitoring_v3.types.TypedValue() + expected_value = monitoring_v3.TypedValue() # TODO: #565 expected_value.double_value = 25.0 * MiB self.assertEqual(time_series.points[0].value, expected_value) def test_create_timeseries_from_distribution(self): """Check for explicit 0-bound bucket for SD export.""" - agg = aggregation_module.DistributionAggregation( - aggregation_type=aggregation_module.Type.DISTRIBUTION) + agg = aggregation_module.DistributionAggregation() view = view_module.View( name="example.org/test_view", @@ -1315,12 +1357,18 @@ def test_create_timeseries_from_distribution(self): include_opencensus=True) self.assertEqual(len(time_series.points), 1) [point] = time_series.points + dv = point.value.distribution_value - self.assertEqual(100, dv.count) - self.assertEqual(825.0, dv.sum_of_squared_deviation) - self.assertEqual([0, 20, 20, 20, 20, 20], dv.bucket_counts) - self.assertEqual([0, 2, 4, 6, 8], - dv.bucket_options.explicit_buckets.bounds) + expected_distb = google.api.distribution_pb2.Distribution( + count=100, + mean=4.5, + sum_of_squared_deviation=825.0, + bucket_options=google.api.distribution_pb2.Distribution.BucketOptions( # noqa + explicit_buckets=google.api.distribution_pb2.Distribution.BucketOptions.Explicit( # noqa + bounds=[0, 2, 4, 6, 8])), + bucket_counts=[0, 20, 20, 20, 20, 20] + ) + self.assertEqual(dv, expected_distb) def test_create_timeseries_multiple_tags(self): """Check that exporter creates timeseries for multiple tag values. @@ -1328,8 +1376,7 @@ def test_create_timeseries_multiple_tags(self): create_time_series_list should return a time series for each set of values in the tag value aggregation map. """ - agg = aggregation_module.CountAggregation( - aggregation_type=aggregation_module.Type.COUNT) + agg = aggregation_module.CountAggregation() view = view_module.View( name="example.org/test_view", @@ -1375,12 +1422,10 @@ def test_create_timeseries_invalid_aggregation(self): v_data = mock.Mock(spec=view_data_module.ViewData) v_data.view.name = "example.org/base_view" v_data.view.columns = [tag_key_module.TagKey('base_key')] - v_data.view.aggregation.aggregation_type = \ - aggregation_module.Type.NONE v_data.start_time = TEST_TIME_STR v_data.end_time = TEST_TIME_STR - base_data = aggregation_data_module.BaseAggregationData(10) + base_data = None v_data.tag_value_aggregation_data_map = { (None,): base_data, } diff --git a/contrib/opencensus-ext-stackdriver/version.py b/contrib/opencensus-ext-stackdriver/version.py index 235cf3f15..671fc3d04 100644 --- a/contrib/opencensus-ext-stackdriver/version.py +++ b/contrib/opencensus-ext-stackdriver/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.5.dev0' +__version__ = '0.9.dev0' diff --git a/contrib/opencensus-ext-threading/opencensus/ext/threading/trace.py b/contrib/opencensus-ext-threading/opencensus/ext/threading/trace.py index 33d0e7522..c2824aa4f 100644 --- a/contrib/opencensus-ext-threading/opencensus/ext/threading/trace.py +++ b/contrib/opencensus-ext-threading/opencensus/ext/threading/trace.py @@ -14,11 +14,10 @@ import logging import threading -from multiprocessing import pool from concurrent import futures +from multiprocessing import pool -from opencensus.trace import execution_context -from opencensus.trace import tracer +from opencensus.trace import execution_context, tracer from opencensus.trace.propagation import binary_format log = logging.getLogger(__name__) diff --git a/contrib/opencensus-ext-threading/setup.py b/contrib/opencensus-ext-threading/setup.py index 6aa3e9425..c9f36102f 100644 --- a/contrib/opencensus-ext-threading/setup.py +++ b/contrib/opencensus-ext-threading/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Threading Integration', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-threading/tests/test_threading_trace.py b/contrib/opencensus-ext-threading/tests/test_threading_trace.py index 47bd19b0f..6985f7b8b 100644 --- a/contrib/opencensus-ext-threading/tests/test_threading_trace.py +++ b/contrib/opencensus-ext-threading/tests/test_threading_trace.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import threading -import mock -from multiprocessing.pool import Pool +import unittest from concurrent.futures import ThreadPoolExecutor +from multiprocessing.pool import Pool + +import mock from opencensus.ext.threading import trace from opencensus.trace import execution_context, tracer diff --git a/contrib/opencensus-ext-zipkin/opencensus/ext/zipkin/trace_exporter/__init__.py b/contrib/opencensus-ext-zipkin/opencensus/ext/zipkin/trace_exporter/__init__.py index 2333e0e0f..096a08701 100644 --- a/contrib/opencensus-ext-zipkin/opencensus/ext/zipkin/trace_exporter/__init__.py +++ b/contrib/opencensus-ext-zipkin/opencensus/ext/zipkin/trace_exporter/__init__.py @@ -20,8 +20,7 @@ import requests from opencensus.common.transports import sync -from opencensus.common.utils import check_str_length -from opencensus.common.utils import timestamp_to_microseconds +from opencensus.common.utils import check_str_length, timestamp_to_microseconds from opencensus.trace import base_exporter DEFAULT_ENDPOINT = '/api/v2/spans' diff --git a/contrib/opencensus-ext-zipkin/setup.py b/contrib/opencensus-ext-zipkin/setup.py index 8611fa5d5..64a161a80 100644 --- a/contrib/opencensus-ext-zipkin/setup.py +++ b/contrib/opencensus-ext-zipkin/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup + from version import __version__ setup( @@ -34,12 +34,14 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='OpenCensus Zipkin Trace Exporter', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.dev0, < 1.0.0', + 'opencensus >= 0.9.dev0, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-zipkin/tests/test_zipkin_exporter.py b/contrib/opencensus-ext-zipkin/tests/test_zipkin_exporter.py index a7032532c..607d13c3c 100644 --- a/contrib/opencensus-ext-zipkin/tests/test_zipkin_exporter.py +++ b/contrib/opencensus-ext-zipkin/tests/test_zipkin_exporter.py @@ -13,9 +13,10 @@ # limitations under the License. import unittest +from datetime import datetime import mock -from datetime import datetime + from opencensus.ext.zipkin import trace_exporter from opencensus.trace import span_context from opencensus.trace import span_data as span_data_module diff --git a/docs/trace/usage.rst b/docs/trace/usage.rst index 0c0db69b2..3c858c84c 100644 --- a/docs/trace/usage.rst +++ b/docs/trace/usage.rst @@ -148,36 +148,36 @@ This example shows how to use the ``GoogleCloudFormatPropagator``: # Serialize header = propagator.to_header(span_context) -Blacklist Paths -~~~~~~~~~~~~~~~ +Excludelist Paths +~~~~~~~~~~~~~~~~~ You can specify which paths you do not want to trace by configuring the -blacklist paths. +excludelist paths. -This example shows how to configure the blacklist to ignore the `_ah/health` endpoint +This example shows how to configure the excludelist to ignore the `_ah/health` endpoint for a Flask application: .. code:: python - from opencensus.trace.ext.flask.flask_middleware import FlaskMiddleware + from opencensus.ext.flask.flask_middleware import FlaskMiddleware app = flask.Flask(__name__) - blacklist_paths = ['_ah/health'] - middleware = FlaskMiddleware(app, blacklist_paths=blacklist_paths) + excludelist_paths = ['_ah/health'] + middleware = FlaskMiddleware(app, excludelist_paths=excludelist_paths) -For Django, you can configure the blacklist in the ``OPENCENSUS_TRACE_PARAMS`` in ``settings.py``: +For Django, you can configure the excludelist in the ``OPENCENSUS_TRACE_PARAMS`` in ``settings.py``: .. code:: python OPENCENSUS_TRACE_PARAMS: { ... - 'BLACKLIST_PATHS': ['_ah/health',], + 'EXCLUDELIST_PATHS': ['_ah/health',], } .. note:: By default the health check path for the App Engine flexible environment is not traced, - but you can turn it on by excluding it from the blacklist setting. + but you can turn it on by excluding it from the excludelist setting. Framework Integration --------------------- @@ -195,7 +195,7 @@ requests will be automatically traced. .. code:: python - from opencensus.trace.ext.flask.flask_middleware import FlaskMiddleware + from opencensus.ext.flask.flask_middleware import FlaskMiddleware app = flask.Flask(__name__) @@ -208,22 +208,13 @@ Django ~~~~~~ For tracing Django requests, you will need to add the following line to -the ``MIDDLEWARE_CLASSES`` section in the Django ``settings.py`` file. +the ``MIDDLEWARE`` section in the Django ``settings.py`` file. .. code:: python - MIDDLEWARE_CLASSES = [ + MIDDLEWARE = [ ... - 'opencensus.trace.ext.django.middleware.OpencensusMiddleware', - ] - -And add this line to the ``INSTALLED_APPS`` section: - -.. code:: python - - INSTALLED_APPS = [ - ... - 'opencensus.trace.ext.django', + 'opencensus.ext.django.middleware.OpencensusMiddleware', ] You can configure the sampler, exporter, propagator using the ``OPENCENSUS_TRACE`` setting in @@ -244,7 +235,7 @@ setting in ``settings.py``: .. code:: python OPENCENSUS_TRACE_PARAMS = { - 'BLACKLIST_PATHS': ['/_ah/health'], + 'EXCLUDELIST_PATHS': ['/_ah/health'], 'GCP_EXPORTER_PROJECT': None, 'SAMPLING_RATE': 0.5, 'SERVICE_NAME': 'my_service', @@ -265,7 +256,7 @@ traced. def main(global_config, **settings): config = Configurator(settings=settings) - config.add_tween('opencensus.trace.ext.pyramid' + config.add_tween('opencensus.ext.pyramid' '.pyramid_middleware.OpenCensusTweenFactory') To configure the sampler, exporter, and propagator, pass the instances @@ -385,7 +376,7 @@ Tests source .tox/py34/bin/activate # Run the unit test - pip install nox-automation + pip install nox # See what's available in the nox suite nox -l diff --git a/examples/stats/helloworld/main.py b/examples/stats/helloworld/main.py index 5f65696ca..0d806cb41 100644 --- a/examples/stats/helloworld/main.py +++ b/examples/stats/helloworld/main.py @@ -16,6 +16,7 @@ import random import time +from pprint import pprint from opencensus.stats import aggregation as aggregation_module from opencensus.stats import measure as measure_module @@ -24,7 +25,6 @@ from opencensus.tags import tag_key as tag_key_module from opencensus.tags import tag_map as tag_map_module from opencensus.tags import tag_value as tag_value_module -from pprint import pprint MiB = 1 << 20 FRONTEND_KEY = tag_key_module.TagKey("my.org/keys/frontend") diff --git a/examples/trace/helloworld/main.py b/examples/trace/helloworld/main.py index 3336fb133..25732a357 100644 --- a/examples/trace/helloworld/main.py +++ b/examples/trace/helloworld/main.py @@ -14,9 +14,7 @@ import time -from opencensus.trace import execution_context -from opencensus.trace import print_exporter -from opencensus.trace import samplers +from opencensus.trace import execution_context, print_exporter, samplers from opencensus.trace.tracer import Tracer diff --git a/noxfile.py b/noxfile.py index 9bc89e790..aa9d3dba6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,9 +14,10 @@ from __future__ import absolute_import -import nox import os +import nox + def _install_dev_packages(session): session.install('-e', 'context/opencensus-context') @@ -24,12 +25,15 @@ def _install_dev_packages(session): session.install('-e', '.') session.install('-e', 'contrib/opencensus-ext-azure') + session.install('-e', 'contrib/opencensus-ext-datadog') session.install('-e', 'contrib/opencensus-ext-dbapi') session.install('-e', 'contrib/opencensus-ext-django') + session.install('-e', 'contrib/opencensus-ext-fastapi') session.install('-e', 'contrib/opencensus-ext-flask') session.install('-e', 'contrib/opencensus-ext-gevent') session.install('-e', 'contrib/opencensus-ext-grpc') session.install('-e', 'contrib/opencensus-ext-httplib') + session.install('-e', 'contrib/opencensus-ext-httpx') session.install('-e', 'contrib/opencensus-ext-jaeger') session.install('-e', 'contrib/opencensus-ext-logging') session.install('-e', 'contrib/opencensus-ext-mysql') @@ -48,14 +52,16 @@ def _install_dev_packages(session): def _install_test_dependencies(session): - session.install('mock') - session.install('pytest') - session.install('pytest-cov') + session.install('mock==3.0.5') + session.install('pytest==4.6.4') + # 842 - Unit tests failing on CI due to failed import for coverage + # Might have something to do with the CircleCI image + # session.install('pytest-cov') session.install('retrying') session.install('unittest2') -@nox.session(python=['2.7', '3.4', '3.5', '3.6']) +@nox.session(python=['2.7', '3.5', '3.6']) def unit(session): """Run the unit test suite.""" @@ -69,13 +75,13 @@ def unit(session): session.run( 'py.test', '--quiet', - '--cov=opencensus', - '--cov=context', - '--cov=contrib', - '--cov-append', - '--cov-config=.coveragerc', - '--cov-report=', - '--cov-fail-under=97', + # '--cov=opencensus', + # '--cov=context', + # '--cov=contrib', + # '--cov-append', + # '--cov-config=.coveragerc', + # '--cov-report=', + # '--cov-fail-under=97', 'tests/unit/', 'context/', 'contrib/', @@ -135,15 +141,15 @@ def lint_setup_py(session): 'python', 'setup.py', 'check', '--restructuredtext', '--strict') -@nox.session(python='3.6') -def cover(session): - """Run the final coverage report. - This outputs the coverage report aggregating coverage from the unit - test runs (not system test runs), and then erases coverage data. - """ - session.install('coverage', 'pytest-cov') - session.run('coverage', 'report', '--show-missing', '--fail-under=100') - session.run('coverage', 'erase') +# @nox.session(python='3.6') +# def cover(session): +# """Run the final coverage report. +# This outputs the coverage report aggregating coverage from the unit +# test runs (not system test runs), and then erases coverage data. +# """ +# session.install('coverage', 'pytest-cov') +# session.run('coverage', 'report', '--show-missing', '--fail-under=100') +# session.run('coverage', 'erase') @nox.session(python='3.6') diff --git a/opencensus/common/backports/__init__.py b/opencensus/common/backports/__init__.py index 02c648c81..46fcc7d3a 100644 --- a/opencensus/common/backports/__init__.py +++ b/opencensus/common/backports/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. import six + import weakref diff --git a/opencensus/common/monitored_resource/aws_identity_doc_utils.py b/opencensus/common/monitored_resource/aws_identity_doc_utils.py index 0b2b7ab98..bfa99c205 100644 --- a/opencensus/common/monitored_resource/aws_identity_doc_utils.py +++ b/opencensus/common/monitored_resource/aws_identity_doc_utils.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opencensus.common.http_handler import get_request import json +from opencensus.common.http_handler import get_request + REGION_KEY = 'region' ACCOUNT_ID_KEY = 'aws_account' INSTANCE_ID_KEY = 'instance_id' diff --git a/opencensus/common/monitored_resource/gcp_metadata_config.py b/opencensus/common/monitored_resource/gcp_metadata_config.py index b33f49d2a..0975d7cf4 100644 --- a/opencensus/common/monitored_resource/gcp_metadata_config.py +++ b/opencensus/common/monitored_resource/gcp_metadata_config.py @@ -14,7 +14,7 @@ from opencensus.common.http_handler import get_request -_GCP_METADATA_URI = 'http://metadata/computeMetadata/v1/' +_GCP_METADATA_URI = 'http://metadata.google.internal/computeMetadata/v1/' _GCP_METADATA_URI_HEADER = {'Metadata-Flavor': 'Google'} # ID of the GCP project associated with this resource, such as "my-project" diff --git a/opencensus/common/monitored_resource/monitored_resource.py b/opencensus/common/monitored_resource/monitored_resource.py index c11b9996d..99e166804 100644 --- a/opencensus/common/monitored_resource/monitored_resource.py +++ b/opencensus/common/monitored_resource/monitored_resource.py @@ -13,10 +13,11 @@ # limitations under the License. from opencensus.common import resource -from opencensus.common.monitored_resource import aws_identity_doc_utils -from opencensus.common.monitored_resource import gcp_metadata_config -from opencensus.common.monitored_resource import k8s_utils - +from opencensus.common.monitored_resource import ( + aws_identity_doc_utils, + gcp_metadata_config, + k8s_utils, +) # Supported environments (resource types) _GCE_INSTANCE = "gce_instance" diff --git a/opencensus/common/resource/__init__.py b/opencensus/common/resource/__init__.py index 0b9507bc1..1a44a82ad 100644 --- a/opencensus/common/resource/__init__.py +++ b/opencensus/common/resource/__init__.py @@ -13,11 +13,10 @@ # limitations under the License. -from copy import copy import logging import os import re - +from copy import copy logger = logging.getLogger(__name__) diff --git a/opencensus/common/schedule/__init__.py b/opencensus/common/schedule/__init__.py index 580f229bd..ed997a620 100644 --- a/opencensus/common/schedule/__init__.py +++ b/opencensus/common/schedule/__init__.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from six.moves import queue + +import logging import threading import time -from six.moves import queue +logger = logging.getLogger(__name__) class PeriodicTask(threading.Thread): @@ -31,11 +34,14 @@ class PeriodicTask(threading.Thread): :param args: The args passed in while calling `function`. :type kwargs: dict - :param args: The kwargs passed in while calling `function`. + :param kwargs: The kwargs passed in while calling `function`. + + :type name: str + :param name: The source of the worker. Used for naming. """ - def __init__(self, interval, function, args=None, kwargs=None): - super(PeriodicTask, self).__init__() + def __init__(self, interval, function, args=None, kwargs=None, name=None): + super(PeriodicTask, self).__init__(name=name) self.interval = interval self.function = function self.args = args or [] @@ -106,6 +112,9 @@ def _gets(self, count, timeout): def gets(self, count, timeout): return tuple(self._gets(count, timeout)) + def is_empty(self): + return not self._queue.qsize() + def flush(self, timeout=None): if self._queue.qsize() == 0: return 0 @@ -118,14 +127,14 @@ def flush(self, timeout=None): return elapsed_time = time.time() - start_time wait_time = timeout and max(timeout - elapsed_time, 0) - if event.wait(timeout): + if event.wait(wait_time): return time.time() - start_time # time taken to flush def put(self, item, block=True, timeout=None): try: self._queue.put(item, block, timeout) except queue.Full: - pass # TODO: log data loss + logger.warning('Queue is full. Dropping telemetry.') def puts(self, items, block=True, timeout=None): if block and timeout is not None: diff --git a/opencensus/common/transports/async_.py b/opencensus/common/transports/async_.py index d6d4e96f0..56e726119 100644 --- a/opencensus/common/transports/async_.py +++ b/opencensus/common/transports/async_.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from six.moves import queue, range + import atexit import logging import threading -from six.moves import queue -from six.moves import range - from opencensus.common.transports import base +from opencensus.trace import execution_context _DEFAULT_GRACE_PERIOD = 5.0 # Seconds _DEFAULT_MAX_BATCH_SIZE = 600 @@ -27,6 +27,8 @@ _WORKER_THREAD_NAME = 'opencensus.common.Worker' _WORKER_TERMINATOR = object() +logger = logging.getLogger(__name__) + class _Worker(object): """A background thread that exports batches of data. @@ -90,6 +92,9 @@ def _thread_main(self): Pulls pending data off the queue and writes them in batches to the specified tracing backend using the exporter. """ + # Indicate that this thread is an exporter thread. + # Used to suppress tracking of requests in this thread + execution_context.set_is_exporter(True) quit_ = False while True: @@ -108,7 +113,7 @@ def _thread_main(self): try: self.exporter.emit(data) except Exception: - logging.exception( + logger.exception( '%s failed to emit data.' 'Dropping %s objects from queue.', self.exporter.__class__.__name__, diff --git a/opencensus/common/transports/sync.py b/opencensus/common/transports/sync.py index aae01218f..8f31d9bfe 100644 --- a/opencensus/common/transports/sync.py +++ b/opencensus/common/transports/sync.py @@ -13,6 +13,7 @@ # limitations under the License. from opencensus.common.transports import base +from opencensus.trace import execution_context class SyncTransport(base.Transport): @@ -20,4 +21,8 @@ def __init__(self, exporter): self.exporter = exporter def export(self, datas): + # Used to suppress tracking of requests in export + execution_context.set_is_exporter(True) self.exporter.emit(datas) + # Reset the context + execution_context.set_is_exporter(False) diff --git a/opencensus/common/version/__init__.py b/opencensus/common/version/__init__.py index ae27d1d16..ebf29a38b 100644 --- a/opencensus/common/version/__init__.py +++ b/opencensus/common/version/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.7.dev0' +__version__ = '0.12.dev0' diff --git a/opencensus/log/__init__.py b/opencensus/log/__init__.py index ddf84f83b..ce4ce5f57 100644 --- a/opencensus/log/__init__.py +++ b/opencensus/log/__init__.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from collections import namedtuple from copy import copy -import logging from opencensus.trace import execution_context - _meta_logger = logging.getLogger(__name__) TRACE_ID_KEY = 'traceId' diff --git a/opencensus/metrics/export/cumulative.py b/opencensus/metrics/export/cumulative.py index 921dd3593..5c2a45aeb 100644 --- a/opencensus/metrics/export/cumulative.py +++ b/opencensus/metrics/export/cumulative.py @@ -14,8 +14,7 @@ import six -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import gauge +from opencensus.metrics.export import gauge, metric_descriptor class CumulativePointLong(gauge.GaugePointLong): diff --git a/opencensus/metrics/export/gauge.py b/opencensus/metrics/export/gauge.py index ff5520809..5ae05985c 100644 --- a/opencensus/metrics/export/gauge.py +++ b/opencensus/metrics/export/gauge.py @@ -12,15 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import OrderedDict -from datetime import datetime import six + import threading +from collections import OrderedDict +from datetime import datetime from opencensus.common import utils -from opencensus.metrics.export import metric -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import metric_producer +from opencensus.metrics.export import ( + metric, + metric_descriptor, + metric_producer, +) from opencensus.metrics.export import point as point_module from opencensus.metrics.export import time_series from opencensus.metrics.export import value as value_module @@ -191,15 +194,17 @@ class DerivedGaugePoint(GaugePoint): :class:`opencensus.metrics.export.cumulative.CumulativePointDouble` :param gauge_point: The underlying `GaugePoint`. """ - def __init__(self, func, gauge_point): + def __init__(self, func, gauge_point, **kwargs): self.gauge_point = gauge_point self.func = utils.get_weakref(func) + self._kwargs = kwargs def __repr__(self): - return ("{}({})" + return ("{}({})({})" .format( type(self).__name__, - self.func() + self.func(), + self._kwargs )) def get_value(self): @@ -213,7 +218,7 @@ def get_value(self): longer exists. """ try: - val = self.func()() + val = self.func()(**self._kwargs) except TypeError: # The underlying function has been GC'd return None @@ -403,13 +408,13 @@ class DerivedGauge(BaseGauge): instead of using this class directly. """ - def _create_time_series(self, label_values, func): + def _create_time_series(self, label_values, func, **kwargs): with self._points_lock: return self.points.setdefault( tuple(label_values), - DerivedGaugePoint(func, self.point_type())) + DerivedGaugePoint(func, self.point_type(), **kwargs)) - def create_time_series(self, label_values, func): + def create_time_series(self, label_values, func, **kwargs): """Create a derived measurement to trac `func`. :type label_values: list(:class:`LabelValue`) @@ -429,7 +434,7 @@ def create_time_series(self, label_values, func): raise ValueError if func is None: raise ValueError - return self._create_time_series(label_values, func) + return self._create_time_series(label_values, func, **kwargs) def create_default_time_series(self, func): """Create the default derived measurement for this gauge. diff --git a/opencensus/metrics/label_value.py b/opencensus/metrics/label_value.py index 88b6e194a..032739d33 100644 --- a/opencensus/metrics/label_value.py +++ b/opencensus/metrics/label_value.py @@ -33,3 +33,10 @@ def __repr__(self): def value(self): """the value for the label""" return self._value + + def __eq__(self, other): + return isinstance(other, LabelValue) and \ + self.value == other.value + + def __hash__(self): + return hash(self.value) diff --git a/opencensus/metrics/transport.py b/opencensus/metrics/transport.py index 8fe93f525..d27b06f32 100644 --- a/opencensus/metrics/transport.py +++ b/opencensus/metrics/transport.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools import logging from opencensus.common import utils from opencensus.common.schedule import PeriodicTask - +from opencensus.trace import execution_context logger = logging.getLogger(__name__) @@ -28,7 +29,7 @@ class TransportError(Exception): pass -class MetricExporterTask(PeriodicTask): +class PeriodicMetricTask(PeriodicTask): """Thread that periodically calls a given function. :type interval: int or float @@ -42,15 +43,27 @@ class MetricExporterTask(PeriodicTask): :type kwargs: dict :param args: The kwargs passed in while calling `function`. + + :type name: str + :param name: The source of the worker. Used for naming. """ daemon = True - def __init__(self, interval=None, function=None, args=None, kwargs=None): + def __init__( + self, + interval=None, + function=None, + args=None, + kwargs=None, + name=None + ): if interval is None: interval = DEFAULT_INTERVAL self.func = function + self.args = args + self.kwargs = kwargs def func(*aa, **kw): try: @@ -58,22 +71,43 @@ def func(*aa, **kw): except TransportError as ex: logger.exception(ex) self.cancel() - except Exception: - logger.exception("Error handling metric export") - - super(MetricExporterTask, self).__init__(interval, func, args, kwargs) - - -def get_exporter_thread(metric_producer, exporter, interval=None): + except Exception as ex: + logger.exception("Error handling metric export: {}".format(ex)) + + super(PeriodicMetricTask, self).__init__( + interval, func, args, kwargs, '{} Worker'.format(name) + ) + + def run(self): + # Indicate that this thread is an exporter thread. + # Used to suppress tracking of requests in this thread + execution_context.set_is_exporter(True) + super(PeriodicMetricTask, self).run() + + def close(self): + try: + # Suppress request tracking on flush + execution_context.set_is_exporter(True) + self.func(*self.args, **self.kwargs) + execution_context.set_is_exporter(False) + except Exception as ex: + logger.exception("Error handling metric flush: {}".format(ex)) + self.cancel() + + +def get_exporter_thread(metric_producers, exporter, interval=None): """Get a running task that periodically exports metrics. Get a `PeriodicTask` that periodically calls: - exporter.export_metrics(metric_producer.get_metrics()) + export(itertools.chain(*all_gets)) + + where all_gets is the concatenation of all metrics produced by the metric + producers in metric_producers, each calling metric_producer.get_metrics() - :type metric_producer: - :class:`opencensus.metrics.export.metric_producer.MetricProducer` - :param exporter: The producer to use to get metrics to export. + :type metric_producers: + list(:class:`opencensus.metrics.export.metric_producer.MetricProducer`) + :param metric_producers: The list of metric producers to use to get metrics :type exporter: :class:`opencensus.stats.base_exporter.MetricsExporter` :param exporter: The exporter to use to export metrics. @@ -85,18 +119,27 @@ def get_exporter_thread(metric_producer, exporter, interval=None): :return: A running thread responsible calling the exporter. """ - weak_get = utils.get_weakref(metric_producer.get_metrics) + weak_gets = [utils.get_weakref(producer.get_metrics) + for producer in metric_producers] weak_export = utils.get_weakref(exporter.export_metrics) def export_all(): - get = weak_get() - if get is None: - raise TransportError("Metric producer is not available") + all_gets = [] + for weak_get in weak_gets: + get = weak_get() + if get is None: + raise TransportError("Metric producer is not available") + all_gets.append(get()) export = weak_export() if export is None: raise TransportError("Metric exporter is not available") - export(get()) - tt = MetricExporterTask(interval, export_all) + export(itertools.chain(*all_gets)) + + tt = PeriodicMetricTask( + interval, + export_all, + name=exporter.__class__.__name__ + ) tt.start() return tt diff --git a/opencensus/stats/aggregation.py b/opencensus/stats/aggregation.py index d9679ba22..acaa564f7 100644 --- a/opencensus/stats/aggregation.py +++ b/opencensus/stats/aggregation.py @@ -14,106 +14,67 @@ import logging -from opencensus.stats import bucket_boundaries +from opencensus.metrics.export.metric_descriptor import MetricDescriptorType from opencensus.stats import aggregation_data - +from opencensus.stats import measure as measure_module logger = logging.getLogger(__name__) -class Type(object): - """ The type of aggregation function used on a View. - - Attributes: - NONE (int): The aggregation type of the view is 'unknown'. - SUM (int): The aggregation type of the view is 'sum'. - COUNT (int): The aggregation type of the view is 'count'. - DISTRIBUTION (int): The aggregation type of the view is 'distribution'. - LASTVALUE (int): The aggregation type of the view is 'lastvalue'. - """ - NONE = 0 - SUM = 1 - COUNT = 2 - DISTRIBUTION = 3 - LASTVALUE = 4 - - -class BaseAggregation(object): - """Aggregation describes how the data collected is aggregated by type of - aggregation and buckets - - :type buckets: list(:class: '~opencensus.stats.bucket_boundaries. - BucketBoundaries') - :param buckets: list of endpoints if the aggregation represents a - distribution - - :type aggregation_type: :class:`~opencensus.stats.aggregation.Type` - :param aggregation_type: represents the type of this aggregation - - """ - def __init__(self, buckets=None, aggregation_type=Type.NONE): - self._aggregation_type = aggregation_type - self._buckets = buckets or [] - - @property - def aggregation_type(self): - """The aggregation type of the current aggregation""" - return self._aggregation_type - - @property - def buckets(self): - """The buckets of the current aggregation""" - return self._buckets - - -class SumAggregation(BaseAggregation): - """Sum Aggregation escribes that data collected and aggregated with this +class SumAggregation(object): + """Sum Aggregation describes that data collected and aggregated with this method will be summed :type sum: int or float - :param sum: the sum of the data collected and aggregated - - - :type aggregation_type: :class:`~opencensus.stats.aggregation.Type` - :param aggregation_type: represents the type of this aggregation + :param sum: the initial sum to be used in the aggregation """ - def __init__(self, sum=None, aggregation_type=Type.SUM): - super(SumAggregation, self).__init__(aggregation_type=aggregation_type) - self._sum = aggregation_data.SumAggregationDataFloat( - sum_data=float(sum or 0)) - self.aggregation_data = self._sum - - @property - def sum(self): - """The sum of the current aggregation""" - return self._sum + def __init__(self, sum=None): + self._initial_sum = sum or 0 + + def new_aggregation_data(self, measure): + """Get a new AggregationData for this aggregation.""" + value_type = MetricDescriptorType.to_type_class( + self.get_metric_type(measure)) + return aggregation_data.SumAggregationData( + value_type=value_type, sum_data=self._initial_sum) + + @staticmethod + def get_metric_type(measure): + """Get the MetricDescriptorType for the metric produced by this + aggregation and measure. + """ + if isinstance(measure, measure_module.MeasureInt): + return MetricDescriptorType.CUMULATIVE_INT64 + if isinstance(measure, measure_module.MeasureFloat): + return MetricDescriptorType.CUMULATIVE_DOUBLE + raise ValueError -class CountAggregation(BaseAggregation): +class CountAggregation(object): """Describes that the data collected and aggregated with this method will be turned into a count value :type count: int - :param count: represents the count of this aggregation - - :type aggregation_type: :class:`~opencensus.stats.aggregation.Type` - :param aggregation_type: represents the type of this aggregation + :param count: the initial count to be used in the aggregation """ - def __init__(self, count=0, aggregation_type=Type.COUNT): - super(CountAggregation, self).__init__( - aggregation_type=aggregation_type) - self._count = aggregation_data.CountAggregationData(count) - self.aggregation_data = self._count + def __init__(self, count=0): + self._initial_count = count - @property - def count(self): - """The count of the current aggregation""" - return self._count + def new_aggregation_data(self, measure=None): + """Get a new AggregationData for this aggregation.""" + return aggregation_data.CountAggregationData(self._initial_count) + + @staticmethod + def get_metric_type(measure): + """Get the MetricDescriptorType for the metric produced by this + aggregation and measure. + """ + return MetricDescriptorType.CUMULATIVE_INT64 -class DistributionAggregation(BaseAggregation): +class DistributionAggregation(object): """Distribution Aggregation indicates that the desired aggregation is a histogram distribution @@ -121,18 +82,9 @@ class DistributionAggregation(BaseAggregation): BucketBoundaries') :param boundaries: the bucket endpoints - :type distribution: histogram - :param distribution: histogram of the values of the population - - :type aggregation_type: :class:`~opencensus.stats.aggregation.Type` - :param aggregation_type: represents the type of this aggregation - """ - def __init__(self, - boundaries=None, - distribution=None, - aggregation_type=Type.DISTRIBUTION): + def __init__(self, boundaries=None): if boundaries: if not all(boundaries[ii] < boundaries[ii + 1] for ii in range(len(boundaries) - 1)): @@ -147,44 +99,46 @@ def __init__(self, ii) boundaries = boundaries[ii:] - super(DistributionAggregation, self).__init__( - buckets=boundaries, aggregation_type=aggregation_type) - self._boundaries = bucket_boundaries.BucketBoundaries(boundaries) - self._distribution = distribution or {} - self.aggregation_data = aggregation_data.DistributionAggregationData( - 0, 0, 0, None, boundaries) + self._boundaries = boundaries - @property - def boundaries(self): - """The boundaries of the current aggregation""" - return self._boundaries + def new_aggregation_data(self, measure=None): + """Get a new AggregationData for this aggregation.""" + return aggregation_data.DistributionAggregationData( + 0, 0, 0, None, self._boundaries) - @property - def distribution(self): - """The distribution of the current aggregation""" - return self._distribution + @staticmethod + def get_metric_type(measure): + """Get the MetricDescriptorType for the metric produced by this + aggregation and measure. + """ + return MetricDescriptorType.CUMULATIVE_DISTRIBUTION -class LastValueAggregation(BaseAggregation): +class LastValueAggregation(object): """Describes that the data collected with this method will overwrite the last recorded value :type value: long - :param value: represents the value of this aggregation - - :type aggregation_type: :class:`~opencensus.stats.aggregation.Type` - :param aggregation_type: represents the type of this aggregation + :param count: the initial value to be used in the aggregation """ - def __init__(self, value=0, aggregation_type=Type.LASTVALUE): - super(LastValueAggregation, self).__init__( - aggregation_type=aggregation_type) - self.aggregation_data = aggregation_data.LastValueAggregationData( - value=value) - self._value = value - - @property - def value(self): - """The current recorded value + def __init__(self, value=0): + self._initial_value = value + + def new_aggregation_data(self, measure): + """Get a new AggregationData for this aggregation.""" + value_type = MetricDescriptorType.to_type_class( + self.get_metric_type(measure)) + return aggregation_data.LastValueAggregationData( + value=self._initial_value, value_type=value_type) + + @staticmethod + def get_metric_type(measure): + """Get the MetricDescriptorType for the metric produced by this + aggregation and measure. """ - return self._value + if isinstance(measure, measure_module.MeasureInt): + return MetricDescriptorType.GAUGE_INT64 + if isinstance(measure, measure_module.MeasureFloat): + return MetricDescriptorType.GAUGE_DOUBLE + raise ValueError diff --git a/opencensus/stats/aggregation_data.py b/opencensus/stats/aggregation_data.py index 1ce6f2fa0..7fa3f9ac5 100644 --- a/opencensus/stats/aggregation_data.py +++ b/opencensus/stats/aggregation_data.py @@ -15,53 +15,26 @@ import copy import logging -from opencensus.metrics.export import point -from opencensus.metrics.export import value +from opencensus.metrics.export import point, value from opencensus.stats import bucket_boundaries - logger = logging.getLogger(__name__) -class BaseAggregationData(object): - """Aggregation Data represents an aggregated value from a collection - - :type aggregation_data: aggregated value - :param aggregation_data: represents the aggregated value from a collection - - """ - - def __init__(self, aggregation_data): - self._aggregation_data = aggregation_data - - @property - def aggregation_data(self): - """The current aggregation data""" - return self._aggregation_data - - def to_point(self, timestamp): - """Get a Point conversion of this aggregation. - - :type timestamp: :class: `datetime.datetime` - :param timestamp: The time to report the point as having been recorded. - - :rtype: :class: `opencensus.metrics.export.point.Point` - :return: a Point with with this aggregation's value and appropriate - value type. - """ - raise NotImplementedError # pragma: NO COVER - - -class SumAggregationDataFloat(BaseAggregationData): +class SumAggregationData(object): """Sum Aggregation Data is the aggregated data for the Sum aggregation - :type sum_data: float - :param sum_data: represents the aggregated sum + :type value_type: class that is either + :class:`opencensus.metrics.export.value.ValueDouble` or + :class:`opencensus.metrics.export.value.ValueLong` + :param value_type: the type of value to be used when creating a point + :type sum_data: int or float + :param sum_data: represents the initial aggregated sum """ - def __init__(self, sum_data): - super(SumAggregationDataFloat, self).__init__(sum_data) + def __init__(self, value_type, sum_data): + self._value_type = value_type self._sum_data = sum_data def __repr__(self): @@ -82,6 +55,11 @@ def sum_data(self): """The current sum data""" return self._sum_data + @property + def value_type(self): + """The value type to use when creating the point""" + return self._value_type + def to_point(self, timestamp): """Get a Point conversion of this aggregation. @@ -89,22 +67,21 @@ def to_point(self, timestamp): :param timestamp: The time to report the point as having been recorded. :rtype: :class: `opencensus.metrics.export.point.Point` - :return: a :class: `opencensus.metrics.export.value.ValueDouble`-valued - Point with value equal to `sum_data`. + :return: a Point with value equal to `sum_data` and of type + `_value_type`. """ - return point.Point(value.ValueDouble(self.sum_data), timestamp) + return point.Point(self._value_type(self.sum_data), timestamp) -class CountAggregationData(BaseAggregationData): +class CountAggregationData(object): """Count Aggregation Data is the count value of aggregated data :type count_data: long - :param count_data: represents the aggregated count + :param count_data: represents the initial aggregated count """ def __init__(self, count_data): - super(CountAggregationData, self).__init__(count_data) self._count_data = count_data def __repr__(self): @@ -137,7 +114,7 @@ def to_point(self, timestamp): return point.Point(value.ValueLong(self.count_data), timestamp) -class DistributionAggregationData(BaseAggregationData): +class DistributionAggregationData(object): """Distribution Aggregation Data refers to the distribution stats of aggregated data @@ -173,7 +150,6 @@ def __init__(self, if exemplars is not None and len(exemplars) != len(bounds) + 1: raise ValueError - super(DistributionAggregationData, self).__init__(mean_data) self._mean_data = mean_data self._count_data = count_data self._sum_of_sqd_deviations = sum_of_sqd_deviations @@ -324,17 +300,21 @@ def to_point(self, timestamp): ) -class LastValueAggregationData(BaseAggregationData): +class LastValueAggregationData(object): """ LastValue Aggregation Data is the value of aggregated data + :type value_type: class that is either + :class:`opencensus.metrics.export.value.ValueDouble` or + :class:`opencensus.metrics.export.value.ValueLong` + :param value_type: the type of value to be used when creating a point :type value: long - :param value: represents the current value + :param value: represents the initial value """ - def __init__(self, value): - super(LastValueAggregationData, self).__init__(value) + def __init__(self, value_type, value): + self._value_type = value_type self._value = value def __repr__(self): @@ -355,6 +335,11 @@ def value(self): """The current value recorded""" return self._value + @property + def value_type(self): + """The value type to use when creating the point""" + return self._value_type + def to_point(self, timestamp): """Get a Point conversion of this aggregation. @@ -362,10 +347,9 @@ def to_point(self, timestamp): :param timestamp: The time to report the point as having been recorded. :rtype: :class: `opencensus.metrics.export.point.Point` - :return: a :class: `opencensus.metrics.export.value.ValueDouble`-valued - Point. + :return: a Point with value of type `_value_type`. """ - return point.Point(value.ValueDouble(self.value), timestamp) + return point.Point(self._value_type(self.value), timestamp) class Exemplar(object): diff --git a/opencensus/stats/measure_to_view_map.py b/opencensus/stats/measure_to_view_map.py index e127272b5..6eab9b2b2 100644 --- a/opencensus/stats/measure_to_view_map.py +++ b/opencensus/stats/measure_to_view_map.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict import copy import logging +from collections import defaultdict from opencensus.stats import metric_utils from opencensus.stats import view_data as view_data_module +logger = logging.getLogger(__name__) + class MeasureToViewMap(object): """Measure To View Map stores a map from names of Measures to @@ -90,13 +92,13 @@ def register_view(self, view, timestamp): # ignore the views that are already registered return else: - logging.warning( + logger.warning( "A different view with the same name is already registered" ) # pragma: NO COVER measure = view.measure registered_measure = self._registered_measures.get(measure.name) if registered_measure is not None and registered_measure != measure: - logging.warning( + logger.warning( "A different measure with the same name is already registered") self._registered_views[view.name] = view if registered_measure is None: diff --git a/opencensus/stats/measurement_map.py b/opencensus/stats/measurement_map.py index 0df811f37..c41ab5ce4 100644 --- a/opencensus/stats/measurement_map.py +++ b/opencensus/stats/measurement_map.py @@ -17,7 +17,6 @@ from opencensus.common import utils from opencensus.tags import TagContext - logger = logging.getLogger(__name__) diff --git a/opencensus/stats/metric_utils.py b/opencensus/stats/metric_utils.py index ebd737cbb..4cd8ec6a7 100644 --- a/opencensus/stats/metric_utils.py +++ b/opencensus/stats/metric_utils.py @@ -16,60 +16,7 @@ """ from opencensus.metrics import label_value -from opencensus.metrics.export import metric -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import time_series -from opencensus.stats import aggregation as aggregation_module -from opencensus.stats import measure as measure_module - -# To check that an aggregation's reported type matches its class -AGGREGATION_TYPE_MAP = { - aggregation_module.Type.SUM: - aggregation_module.SumAggregation, - aggregation_module.Type.COUNT: - aggregation_module.CountAggregation, - aggregation_module.Type.DISTRIBUTION: - aggregation_module.DistributionAggregation, - aggregation_module.Type.LASTVALUE: - aggregation_module.LastValueAggregation, -} - - -def get_metric_type(measure, aggregation): - """Get the corresponding metric type for the given stats type. - - :type measure: (:class: '~opencensus.stats.measure.BaseMeasure') - :param measure: the measure for which to find a metric type - - :type aggregation: (:class: - '~opencensus.stats.aggregation.BaseAggregation') - :param aggregation: the aggregation for which to find a metric type - """ - if aggregation.aggregation_type == aggregation_module.Type.NONE: - raise ValueError("aggregation type must not be NONE") - assert isinstance(aggregation, - AGGREGATION_TYPE_MAP[aggregation.aggregation_type]) - - if aggregation.aggregation_type == aggregation_module.Type.SUM: - if isinstance(measure, measure_module.MeasureInt): - return metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64 - elif isinstance(measure, measure_module.MeasureFloat): - return metric_descriptor.MetricDescriptorType.CUMULATIVE_DOUBLE - else: - raise ValueError - elif aggregation.aggregation_type == aggregation_module.Type.COUNT: - return metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64 - elif aggregation.aggregation_type == aggregation_module.Type.DISTRIBUTION: - return metric_descriptor.MetricDescriptorType.CUMULATIVE_DISTRIBUTION - elif aggregation.aggregation_type == aggregation_module.Type.LASTVALUE: - if isinstance(measure, measure_module.MeasureInt): - return metric_descriptor.MetricDescriptorType.GAUGE_INT64 - elif isinstance(measure, measure_module.MeasureFloat): - return metric_descriptor.MetricDescriptorType.GAUGE_DOUBLE - else: - raise ValueError - else: - raise AssertionError # pragma: NO COVER +from opencensus.metrics.export import metric, metric_descriptor, time_series def is_gauge(md_type): diff --git a/opencensus/stats/stats_recorder.py b/opencensus/stats/stats_recorder.py index 8ea04daf8..f74b6d3b7 100644 --- a/opencensus/stats/stats_recorder.py +++ b/opencensus/stats/stats_recorder.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opencensus.stats.measurement_map import MeasurementMap -from opencensus.stats.measure_to_view_map import MeasureToViewMap from opencensus.stats import execution_context +from opencensus.stats.measure_to_view_map import MeasureToViewMap +from opencensus.stats.measurement_map import MeasurementMap class StatsRecorder(object): diff --git a/opencensus/stats/view.py b/opencensus/stats/view.py index cb5338a89..2bd3470ad 100644 --- a/opencensus/stats/view.py +++ b/opencensus/stats/view.py @@ -17,7 +17,6 @@ from opencensus.metrics import label_key from opencensus.metrics.export import metric_descriptor -from opencensus.stats import metric_utils class View(object): @@ -78,6 +77,14 @@ def aggregation(self): """the aggregation of the current view""" return self._aggregation + def new_aggregation_data(self): + """Get a new AggregationData for this view. + + :rtype: :class: `opencensus.status.aggregation_data.AggregationData` + :return: A new AggregationData. + """ + return self._aggregation.new_aggregation_data(self.measure) + def get_metric_descriptor(self): """Get a MetricDescriptor for this view. @@ -93,8 +100,7 @@ def get_metric_descriptor(self): self.name, self.description, self.measure.unit, - metric_utils.get_metric_type(self.measure, - self.aggregation), + self.aggregation.get_metric_type(self.measure), # TODO: add label key description [label_key.LabelKey(tk, "") for tk in self.columns]) return self._metric_descriptor diff --git a/opencensus/stats/view_data.py b/opencensus/stats/view_data.py index 47adaa33d..f344616bc 100644 --- a/opencensus/stats/view_data.py +++ b/opencensus/stats/view_data.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy - from opencensus.common import utils @@ -92,7 +90,7 @@ def record(self, context, value, timestamp, attachments=None): columns=self.view.columns) tuple_vals = tuple(tag_values) if tuple_vals not in self.tag_value_aggregation_data_map: - self.tag_value_aggregation_data_map[tuple_vals] = copy.deepcopy( - self.view.aggregation.aggregation_data) + self.tag_value_aggregation_data_map[tuple_vals] = \ + self.view.new_aggregation_data() self.tag_value_aggregation_data_map.get(tuple_vals).\ add_sample(value, timestamp, attachments) diff --git a/opencensus/stats/view_manager.py b/opencensus/stats/view_manager.py index 2afaea22c..3b118a11d 100644 --- a/opencensus/stats/view_manager.py +++ b/opencensus/stats/view_manager.py @@ -13,8 +13,8 @@ # limitations under the License. from opencensus.common import utils -from opencensus.stats.measure_to_view_map import MeasureToViewMap from opencensus.stats import execution_context +from opencensus.stats.measure_to_view_map import MeasureToViewMap class ViewManager(object): diff --git a/opencensus/tags/__init__.py b/opencensus/tags/__init__.py index 771d4c255..e4a064695 100644 --- a/opencensus/tags/__init__.py +++ b/opencensus/tags/__init__.py @@ -15,8 +15,8 @@ from opencensus.common.runtime_context import RuntimeContext from opencensus.tags.tag import Tag from opencensus.tags.tag_key import TagKey -from opencensus.tags.tag_value import TagValue from opencensus.tags.tag_map import TagMap +from opencensus.tags.tag_value import TagValue __all__ = ['Tag', 'TagContext', 'TagKey', 'TagValue', 'TagMap'] diff --git a/opencensus/tags/propagation/binary_serializer.py b/opencensus/tags/propagation/binary_serializer.py index 01ed717f2..49c84925a 100644 --- a/opencensus/tags/propagation/binary_serializer.py +++ b/opencensus/tags/propagation/binary_serializer.py @@ -14,9 +14,10 @@ # -*- coding: utf-8 -*- -import logging import six +import logging + from google.protobuf.internal.encoder import _VarintBytes from opencensus.tags import tag_map as tag_map_module diff --git a/opencensus/tags/tag.py b/opencensus/tags/tag.py index e5044c04d..4f78ac833 100644 --- a/opencensus/tags/tag.py +++ b/opencensus/tags/tag.py @@ -13,6 +13,7 @@ # limitations under the License. from collections import namedtuple + from opencensus.tags.tag_key import TagKey from opencensus.tags.tag_value import TagValue diff --git a/opencensus/trace/__init__.py b/opencensus/trace/__init__.py index b1bbded81..a55903b90 100644 --- a/opencensus/trace/__init__.py +++ b/opencensus/trace/__init__.py @@ -14,5 +14,4 @@ from opencensus.trace.span import Span - __all__ = ['Span'] diff --git a/opencensus/trace/attributes_helper.py b/opencensus/trace/attributes_helper.py index 235eaa7e8..f8221951e 100644 --- a/opencensus/trace/attributes_helper.py +++ b/opencensus/trace/attributes_helper.py @@ -23,6 +23,8 @@ 'HTTP_CLIENT_REGION': 'http.client_region', 'HTTP_HOST': 'http.host', 'HTTP_METHOD': 'http.method', + 'HTTP_PATH': 'http.path', + 'HTTP_ROUTE': 'http.route', 'HTTP_REDIRECTED_URL': 'http.redirected_url', 'HTTP_REQUEST_SIZE': 'http.request_size', 'HTTP_RESPONSE_SIZE': 'http.response_size', diff --git a/opencensus/trace/base_span.py b/opencensus/trace/base_span.py index 4eacccde8..cabf5df20 100644 --- a/opencensus/trace/base_span.py +++ b/opencensus/trace/base_span.py @@ -80,6 +80,14 @@ def add_link(self, link): """ raise NotImplementedError + def set_status(self, status): + """Sets span status. + + :type code: :class: `~opencensus.trace.status.Status` + :param code: A Status object. + """ + raise NotImplementedError + def start(self): """Set the start time for a span.""" raise NotImplementedError diff --git a/opencensus/trace/blank_span.py b/opencensus/trace/blank_span.py index d5b09fbf1..8911bf358 100644 --- a/opencensus/trace/blank_span.py +++ b/opencensus/trace/blank_span.py @@ -136,6 +136,14 @@ def add_link(self, link): """ pass + def set_status(self, status): + """No-op implementation of this method. + + :type code: :class: `~opencensus.trace.status.Status` + :param code: A Status object. + """ + pass + def start(self): """No-op implementation of this method.""" pass diff --git a/opencensus/trace/exceptions_status.py b/opencensus/trace/exceptions_status.py new file mode 100644 index 000000000..00542d0c6 --- /dev/null +++ b/opencensus/trace/exceptions_status.py @@ -0,0 +1,25 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.rpc import code_pb2 + +from opencensus.trace.status import Status + +CANCELLED = Status(code_pb2.CANCELLED) +INVALID_URL = Status(code_pb2.INVALID_ARGUMENT, message='invalid URL') +TIMEOUT = Status(code_pb2.DEADLINE_EXCEEDED, message='request timed out') + + +def unknown(exception): + return Status.from_exception(exception) diff --git a/opencensus/trace/execution_context.py b/opencensus/trace/execution_context.py index 89344c6f4..eaa33979c 100644 --- a/opencensus/trace/execution_context.py +++ b/opencensus/trace/execution_context.py @@ -17,9 +17,18 @@ _attrs_slot = RuntimeContext.register_slot('attrs', lambda: {}) _current_span_slot = RuntimeContext.register_slot('current_span', None) +_exporter_slot = RuntimeContext.register_slot('is_exporter', False) _tracer_slot = RuntimeContext.register_slot('tracer', noop_tracer.NoopTracer()) +def is_exporter(): + return RuntimeContext.is_exporter + + +def set_is_exporter(is_exporter): + RuntimeContext.is_exporter = is_exporter + + def get_opencensus_tracer(): """Get the opencensus tracer from runtime context.""" return RuntimeContext.tracer diff --git a/opencensus/trace/file_exporter.py b/opencensus/trace/file_exporter.py index 22eb9b738..baaeaddfc 100644 --- a/opencensus/trace/file_exporter.py +++ b/opencensus/trace/file_exporter.py @@ -17,8 +17,7 @@ import json from opencensus.common.transports import sync -from opencensus.trace import base_exporter -from opencensus.trace import span_data +from opencensus.trace import base_exporter, span_data DEFAULT_FILENAME = 'opencensus-traces.json' diff --git a/opencensus/trace/integrations.py b/opencensus/trace/integrations.py new file mode 100644 index 000000000..59934881d --- /dev/null +++ b/opencensus/trace/integrations.py @@ -0,0 +1,52 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + +_INTEGRATIONS_BIT_MASK = 0 +_INTEGRATIONS_LOCK = threading.Lock() + + +class _Integrations: + NONE = 0 + DJANGO = 1 + FLASK = 2 + GOOGLE_CLOUD = 4 + HTTP_LIB = 8 + LOGGING = 16 + MYSQL = 32 + POSTGRESQL = 64 + PYMONGO = 128 + PYMYSQL = 256 + PYRAMID = 512 + REQUESTS = 1024 + SQLALCHEMY = 2056 + HTTPX = 16777216 + FASTAPI = 4194304 + + +def get_integrations(): + return _INTEGRATIONS_BIT_MASK + + +def add_integration(integration): + with _INTEGRATIONS_LOCK: + global _INTEGRATIONS_BIT_MASK # pylint: disable=global-statement + _INTEGRATIONS_BIT_MASK |= integration + + +def remove_intregration(integration): + with _INTEGRATIONS_LOCK: + global _INTEGRATIONS_BIT_MASK # pylint: disable=global-statement + _INTEGRATIONS_BIT_MASK &= ~integration diff --git a/opencensus/trace/logging_exporter.py b/opencensus/trace/logging_exporter.py index f232e3512..e727f602c 100644 --- a/opencensus/trace/logging_exporter.py +++ b/opencensus/trace/logging_exporter.py @@ -17,8 +17,7 @@ import logging from opencensus.common.transports import sync -from opencensus.trace import base_exporter -from opencensus.trace import span_data +from opencensus.trace import base_exporter, span_data class LoggingExporter(base_exporter.Exporter): diff --git a/opencensus/trace/propagation/b3_format.py b/opencensus/trace/propagation/b3_format.py index cd175efa5..b60dcd8a1 100644 --- a/opencensus/trace/propagation/b3_format.py +++ b/opencensus/trace/propagation/b3_format.py @@ -13,7 +13,7 @@ # limitations under the License. -from opencensus.trace.span_context import SpanContext, INVALID_SPAN_ID +from opencensus.trace.span_context import INVALID_SPAN_ID, SpanContext from opencensus.trace.trace_options import TraceOptions _STATE_HEADER_KEY = 'b3' diff --git a/opencensus/trace/propagation/google_cloud_format.py b/opencensus/trace/propagation/google_cloud_format.py index 83cd8141b..90078df1a 100644 --- a/opencensus/trace/propagation/google_cloud_format.py +++ b/opencensus/trace/propagation/google_cloud_format.py @@ -19,7 +19,7 @@ from opencensus.trace.trace_options import TraceOptions _TRACE_CONTEXT_HEADER_NAME = 'X-Cloud-Trace-Context' -_TRACE_CONTEXT_HEADER_FORMAT = r'([0-9a-f]{32})(\/([0-9a-f]{16}))?(;o=(\d+))?' +_TRACE_CONTEXT_HEADER_FORMAT = r'([0-9a-f]{32})(\/([\d]{0,20}))?(;o=(\d+))?' _TRACE_CONTEXT_HEADER_RE = re.compile(_TRACE_CONTEXT_HEADER_FORMAT) _TRACE_ID_DELIMETER = '/' _SPAN_ID_DELIMETER = ';' @@ -62,6 +62,9 @@ def from_header(self, header): if trace_options is None: trace_options = 1 + if span_id: + span_id = '{:016x}'.format(int(span_id)) + span_context = SpanContext( trace_id=trace_id, span_id=span_id, @@ -88,7 +91,6 @@ def from_headers(self, headers): header = headers.get(_TRACE_CONTEXT_HEADER_NAME) if header is None: return SpanContext() - header = str(header.encode('utf-8')) return self.from_header(header) def to_header(self, span_context): @@ -107,7 +109,7 @@ def to_header(self, span_context): header = '{}/{};o={}'.format( trace_id, - span_id, + int(span_id, 16), int(trace_options)) return header diff --git a/opencensus/trace/propagation/trace_context_http_header_format.py b/opencensus/trace/propagation/trace_context_http_header_format.py index 4b7afa100..095935a2e 100644 --- a/opencensus/trace/propagation/trace_context_http_header_format.py +++ b/opencensus/trace/propagation/trace_context_http_header_format.py @@ -14,10 +14,11 @@ import re +from opencensus.trace.propagation.tracestate_string_format import ( + TracestateStringFormatter, +) from opencensus.trace.span_context import SpanContext from opencensus.trace.trace_options import TraceOptions -from opencensus.trace.propagation.tracestate_string_format \ - import TracestateStringFormatter _TRACEPARENT_HEADER_NAME = 'traceparent' _TRACESTATE_HEADER_NAME = 'tracestate' diff --git a/opencensus/trace/propagation/tracestate_string_format.py b/opencensus/trace/propagation/tracestate_string_format.py index da2882880..9a7af86a5 100644 --- a/opencensus/trace/propagation/tracestate_string_format.py +++ b/opencensus/trace/propagation/tracestate_string_format.py @@ -13,9 +13,8 @@ # limitations under the License. import re -from opencensus.trace.tracestate import Tracestate -from opencensus.trace.tracestate import _KEY_FORMAT -from opencensus.trace.tracestate import _VALUE_FORMAT + +from opencensus.trace.tracestate import _KEY_FORMAT, _VALUE_FORMAT, Tracestate _DELIMITER_FORMAT = '[ \t]*,[ \t]*' _MEMBER_FORMAT = '(%s)(=)(%s)' % (_KEY_FORMAT, _VALUE_FORMAT) diff --git a/opencensus/trace/span.py b/opencensus/trace/span.py index a8e9164b8..4e872c060 100644 --- a/opencensus/trace/span.py +++ b/opencensus/trace/span.py @@ -19,11 +19,10 @@ from collections import MutableMapping from collections import Sequence -from collections import OrderedDict -from collections import deque +import threading +from collections import OrderedDict, deque from datetime import datetime from itertools import chain -import threading from opencensus.common import utils from opencensus.trace import attributes as attributes_module @@ -35,7 +34,6 @@ from opencensus.trace.span_context import generate_span_id from opencensus.trace.tracers import base - # https://github.com/census-instrumentation/opencensus-specs/blob/master/trace/TraceConfig.md # noqa MAX_NUM_ATTRIBUTES = 32 MAX_NUM_ANNOTATIONS = 32 @@ -264,9 +262,13 @@ def __init__( else: self.links = BoundedList.from_seq(MAX_NUM_LINKS, links) + if status is None: + self.status = status_module.Status.as_ok() + else: + self.status = status + self.span_id = span_id self.stack_trace = stack_trace - self.status = status self.same_process_as_parent_span = same_process_as_parent_span self._child_spans = [] self.context_tracer = context_tracer @@ -346,6 +348,18 @@ def add_link(self, link): raise TypeError("Type Error: received {}, but requires Link.". format(type(link).__name__)) + def set_status(self, status): + """Sets span status. + + :type code: :class: `~opencensus.trace.status.Status` + :param code: A Status object. + """ + if isinstance(status, status_module.Status): + self.status = status + else: + raise TypeError("Type Error: received {}, but requires Status.". + format(type(status).__name__)) + def start(self): """Set the start time for a span.""" self.start_time = utils.to_iso_str() @@ -356,7 +370,7 @@ def finish(self): def __iter__(self): """Iterate through the span tree.""" - for span in chain(*(map(iter, self.children))): + for span in chain.from_iterable(map(iter, self.children)): yield span yield self diff --git a/opencensus/trace/span_context.py b/opencensus/trace/span_context.py index 6cd659e17..24fc216f1 100644 --- a/opencensus/trace/span_context.py +++ b/opencensus/trace/span_context.py @@ -14,10 +14,11 @@ """SpanContext encapsulates the current context within the request's trace.""" -import logging -import re import six + +import logging import random +import re from opencensus.trace import trace_options as trace_options_module @@ -42,7 +43,6 @@ class SpanContext(object): :type span_id: str :param span_id: (Optional) Identifier for the span, unique within a trace. - If not given, will generate one automatically. :type trace_options: :class: `~opencensus.trace.trace_options.TraceOptions` :param trace_options: (Optional) TraceOptions indicates 8 trace options. diff --git a/opencensus/trace/stack_trace.py b/opencensus/trace/stack_trace.py index 17975d311..30a810b68 100644 --- a/opencensus/trace/stack_trace.py +++ b/opencensus/trace/stack_trace.py @@ -181,7 +181,7 @@ def generate_hash_id(): def generate_hash_id_from_traceback(tb): - m = hashlib.md5() + m = hashlib.md5() # nosec for tb_line in traceback.format_tb(tb): m.update(tb_line.encode('utf-8')) # truncate the hash for easier compatibility with StackDriver, diff --git a/opencensus/trace/status.py b/opencensus/trace/status.py index 26e7fd53d..612425495 100644 --- a/opencensus/trace/status.py +++ b/opencensus/trace/status.py @@ -39,17 +39,31 @@ class Status(object): See: https://cloud.google.com/trace/docs/reference/v2/ rest/v2/Status#FIELDS.details """ - def __init__(self, code, message, details=None): + def __init__(self, code, message=None, details=None): self.code = code self.message = message self.details = details + @property + def canonical_code(self): + return self.code + + @property + def description(self): + return self.message + + @property + def is_ok(self): + return self.canonical_code == code_pb2.OK + def format_status_json(self): """Convert a Status object to json format.""" status_json = {} - status_json['code'] = self.code - status_json['message'] = self.message + status_json['code'] = self.canonical_code + + if self.description is not None: + status_json['message'] = self.description if self.details is not None: status_json['details'] = self.details @@ -62,3 +76,9 @@ def from_exception(cls, exc): code=code_pb2.UNKNOWN, message=str(exc) ) + + @classmethod + def as_ok(cls): + return cls( + code=code_pb2.OK, + ) diff --git a/opencensus/trace/tracer.py b/opencensus/trace/tracer.py index 3981b3db7..71b7a4646 100644 --- a/opencensus/trace/tracer.py +++ b/opencensus/trace/tracer.py @@ -12,13 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opencensus.trace import execution_context -from opencensus.trace import print_exporter -from opencensus.trace import samplers +from opencensus.trace import execution_context, print_exporter, samplers from opencensus.trace.propagation import trace_context_http_header_format from opencensus.trace.span_context import SpanContext -from opencensus.trace.tracers import context_tracer -from opencensus.trace.tracers import noop_tracer +from opencensus.trace.tracers import context_tracer, noop_tracer class Tracer(object): diff --git a/opencensus/trace/tracers/context_tracer.py b/opencensus/trace/tracers/context_tracer.py index 331a26661..4a6153a92 100644 --- a/opencensus/trace/tracers/context_tracer.py +++ b/opencensus/trace/tracers/context_tracer.py @@ -15,11 +15,10 @@ import logging import threading -from opencensus.trace import execution_context -from opencensus.trace.span_context import SpanContext -from opencensus.trace import print_exporter +from opencensus.trace import execution_context, print_exporter from opencensus.trace import span as trace_span from opencensus.trace import span_data as span_data_module +from opencensus.trace.span_context import SpanContext from opencensus.trace.tracers import base diff --git a/opencensus/trace/tracers/noop_tracer.py b/opencensus/trace/tracers/noop_tracer.py index 2a4b326ff..b61fab53c 100644 --- a/opencensus/trace/tracers/noop_tracer.py +++ b/opencensus/trace/tracers/noop_tracer.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opencensus.trace.tracers import base from opencensus.trace import blank_span as trace_span -from opencensus.trace.span_context import SpanContext from opencensus.trace import trace_options +from opencensus.trace.span_context import SpanContext +from opencensus.trace.tracers import base class NoopTracer(base.Tracer): diff --git a/opencensus/trace/tracestate.py b/opencensus/trace/tracestate.py index 8e328140e..f9110eec9 100644 --- a/opencensus/trace/tracestate.py +++ b/opencensus/trace/tracestate.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import OrderedDict import re +from collections import OrderedDict _KEY_WITHOUT_VENDOR_FORMAT = r'[a-z][_0-9a-z\-\*\/]{0,255}' _KEY_WITH_VENDOR_FORMAT = \ diff --git a/opencensus/trace/utils.py b/opencensus/trace/utils.py index 1a39c8c0e..5b9be3473 100644 --- a/opencensus/trace/utils.py +++ b/opencensus/trace/utils.py @@ -14,12 +14,15 @@ import re +from google.rpc import code_pb2 + from opencensus.trace import execution_context +from opencensus.trace.status import Status -# By default the blacklist urls are not tracing, currently just include the +# By default the excludelist urls are not tracing, currently just include the # health check url. The paths are literal string matched instead of regular # expressions. Do not include the '/' at the beginning of the path. -DEFAULT_BLACKLIST_PATHS = [ +DEFAULT_EXCLUDELIST_PATHS = [ '_ah/health', ] @@ -39,20 +42,20 @@ def get_func_name(func): return func_name -def disable_tracing_url(url, blacklist_paths=None): - """Disable tracing on the provided blacklist paths, by default not tracing +def disable_tracing_url(url, excludelist_paths=None): + """Disable tracing on the provided excludelist paths, by default not tracing the health check request. - If the url path starts with the blacklisted path, return True. + If the url path starts with the excludelisted path, return True. - :type blacklist_paths: list - :param blacklist_paths: Paths that not tracing. + :type excludelist_paths: list + :param excludelist_paths: Paths that not tracing. :rtype: bool :returns: True if not tracing, False if tracing. """ - if blacklist_paths is None: - blacklist_paths = DEFAULT_BLACKLIST_PATHS + if excludelist_paths is None: + excludelist_paths = DEFAULT_EXCLUDELIST_PATHS # Remove the 'https?|ftp://' if exists url = re.sub(URL_PATTERN, '', url) @@ -60,36 +63,66 @@ def disable_tracing_url(url, blacklist_paths=None): # Split the url by the first '/' and get the path part url_path = url.split('/', 1)[1] - for path in blacklist_paths: + for path in excludelist_paths: if url_path.startswith(path): return True return False -def disable_tracing_hostname(url, blacklist_hostnames=None): - """Disable tracing for the provided blacklist URLs, by default not tracing +def disable_tracing_hostname(url, excludelist_hostnames=None): + """Disable tracing for the provided excludelist URLs, by default not tracing the exporter url. - If the url path starts with the blacklisted path, return True. + If the url path starts with the excludelisted path, return True. - :type blacklist_hostnames: list - :param blacklist_hostnames: URL that not tracing. + :type excludelist_hostnames: list + :param excludelist_hostnames: URL that not tracing. :rtype: bool :returns: True if not tracing, False if tracing. """ - if blacklist_hostnames is None: + if excludelist_hostnames is None: # Exporter host_name are not traced by default _tracer = execution_context.get_opencensus_tracer() try: - blacklist_hostnames = [ + excludelist_hostnames = [ '{}:{}'.format( _tracer.exporter.host_name, _tracer.exporter.port ) ] except(AttributeError): - blacklist_hostnames = [] + excludelist_hostnames = [] + + return url in excludelist_hostnames + - return url in blacklist_hostnames +def status_from_http_code(http_code): + """Returns equivalent status from http status code + based on OpenCensus specs. + + :type http_code: int + :param http_code: HTTP request status code. + + :rtype: int + :returns: A instance of :class: `~opencensus.trace.status.Status`. + """ + if http_code <= 199: + return Status(code_pb2.UNKNOWN) + + if http_code <= 399: + return Status(code_pb2.OK) + + grpc_code = { + 400: code_pb2.INVALID_ARGUMENT, + 401: code_pb2.UNAUTHENTICATED, + 403: code_pb2.PERMISSION_DENIED, + 404: code_pb2.NOT_FOUND, + 429: code_pb2.RESOURCE_EXHAUSTED, + 501: code_pb2.UNIMPLEMENTED, + 503: code_pb2.UNAVAILABLE, + 504: code_pb2.DEADLINE_EXCEEDED, + }.get(http_code, code_pb2.UNKNOWN) + + return Status(grpc_code) diff --git a/scripts/pylint.sh b/scripts/pylint.sh new file mode 100644 index 000000000..27c6d62b8 --- /dev/null +++ b/scripts/pylint.sh @@ -0,0 +1,25 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ev + +# Run pylint on directories +function pylint_dir { + python -m pip install --upgrade pylint + pylint $(find context/ contrib opencensus/ tests/ examples/ -type f -name "*.py") + # TODO fix lint errors + return $? +} + +pylint_dir diff --git a/scripts/twine_upload.sh b/scripts/twine_upload.sh index 68eddb907..b505a5bb5 100755 --- a/scripts/twine_upload.sh +++ b/scripts/twine_upload.sh @@ -41,5 +41,5 @@ done # Upload the distributions. for p in dist/* ; do - twine upload $p + twine upload --skip-existing $p done diff --git a/setup.py b/setup.py index 0b9761871..ae24e21bd 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,7 @@ # limitations under the License. """A setup module for OpenCensus Instrumentation Library""" -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup exec(open('opencensus/common/version/__init__.py').read()) @@ -36,13 +35,17 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], description='A stats collection and distributed tracing framework', include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus-context == 0.2.dev0', - 'google-api-core >= 1.0.0, < 2.0.0', + 'opencensus-context >= 0.2.dev0', + 'google-api-core >= 1.0.0, < 2.0.0; python_version<"3.6"', + 'google-api-core >= 1.0.0, < 3.0.0; python_version>="3.6"', + "six ~= 1.16", ], extras_require={}, license='Apache-2.0', diff --git a/tests/system/stats/stackdriver/stackdriver_stats_test.py b/tests/system/stats/stackdriver/stackdriver_stats_test.py index 59857b7d6..c92dd726f 100644 --- a/tests/system/stats/stackdriver/stackdriver_stats_test.py +++ b/tests/system/stats/stackdriver/stackdriver_stats_test.py @@ -17,8 +17,8 @@ import sys import time -from google.cloud import monitoring_v3 import mock +from google.cloud import monitoring_v3 from opencensus.ext.stackdriver import stats_exporter as stackdriver from opencensus.metrics import transport @@ -46,7 +46,7 @@ class TestBasicStats(unittest.TestCase): def check_sd_md(self, exporter, view_description): """Check that the metric descriptor was written to stackdriver.""" - name = exporter.client.project_path(PROJECT) + name = exporter.client.common_project_path(PROJECT) list_metrics_descriptors = exporter.client.list_metric_descriptors( name) @@ -66,15 +66,26 @@ def setUp(self): patcher.start() self.addCleanup(patcher.stop) + def tearDown(self): + suffix = str(os.getgid()) + + cli = monitoring_v3.MetricServiceClient() + for md in cli.list_metric_descriptors('projects/{}'.format(PROJECT)): + if "OpenCensus" in md.name and suffix in md.name: + try: + cli.delete_metric_descriptor(md.name) + except Exception: + pass + def test_stats_record_sync(self): - # We are using sufix in order to prevent cached objects - sufix = str(os.getgid()) + # We are using suffix in order to prevent cached objects + suffix = str(os.getgid()) - tag_key = "SampleKeySyncTest%s" % sufix - measure_name = "SampleMeasureNameSyncTest%s" % sufix - measure_description = "SampleDescriptionSyncTest%s" % sufix - view_name = "SampleViewNameSyncTest%s" % sufix - view_description = "SampleViewDescriptionSyncTest%s" % sufix + tag_key = "SampleKeySyncTest%s" % suffix + measure_name = "SampleMeasureNameSyncTest%s" % suffix + measure_description = "SampleDescriptionSyncTest%s" % suffix + view_name = "SampleViewNameSyncTest%s" % suffix + view_description = "SampleViewDescriptionSyncTest%s" % suffix FRONTEND_KEY = tag_key_module.TagKey(tag_key) VIDEO_SIZE_MEASURE = measure_module.MeasureInt( @@ -119,14 +130,14 @@ def test_stats_record_sync(self): self.check_sd_md(exporter, view_description) def test_stats_record_async(self): - # We are using sufix in order to prevent cached objects - sufix = str(os.getpid()) - - tag_key = "SampleKeyAsyncTest%s" % sufix - measure_name = "SampleMeasureNameAsyncTest%s" % sufix - measure_description = "SampleDescriptionAsyncTest%s" % sufix - view_name = "SampleViewNameAsyncTest%s" % sufix - view_description = "SampleViewDescriptionAsyncTest%s" % sufix + # We are using suffix in order to prevent cached objects + suffix = str(os.getpid()) + + tag_key = "SampleKeyAsyncTest%s" % suffix + measure_name = "SampleMeasureNameAsyncTest%s" % suffix + measure_description = "SampleDescriptionAsyncTest%s" % suffix + view_name = "SampleViewNameAsyncTest%s" % suffix + view_description = "SampleViewDescriptionAsyncTest%s" % suffix FRONTEND_KEY_ASYNC = tag_key_module.TagKey(tag_key) VIDEO_SIZE_MEASURE_ASYNC = measure_module.MeasureInt( diff --git a/tests/system/trace/basic_trace/basic_trace_system_test.py b/tests/system/trace/basic_trace/basic_trace_system_test.py index c6288d3db..86a895ed0 100644 --- a/tests/system/trace/basic_trace/basic_trace_system_test.py +++ b/tests/system/trace/basic_trace/basic_trace_system_test.py @@ -33,7 +33,8 @@ def test_tracer(self): span_id = '6e0c63257de34c92' trace_option = 1 - trace_header = '{}/{};o={}'.format(trace_id, span_id, trace_option) + trace_header = '{}/{};o={}'.format( + trace_id, int(span_id, 16), trace_option) sampler = samplers.AlwaysOnSampler() exporter = file_exporter.FileExporter() diff --git a/tests/system/trace/django/app/settings.py b/tests/system/trace/django/app/settings.py index b77d2b2b5..c60837af9 100644 --- a/tests/system/trace/django/app/settings.py +++ b/tests/system/trace/django/app/settings.py @@ -16,8 +16,6 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os -import django - BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SECRET_KEY = 'secret_key_for_test' @@ -31,29 +29,13 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'opencensus.ext.django', ) -if django.VERSION >= (1, 10): - MIDDLEWARE = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', - 'opencensus.ext.django.middleware.OpencensusMiddleware', - ) - -# Middleware interface for Django version before 1.10 -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.security.SecurityMiddleware', @@ -89,7 +71,7 @@ 'PROPAGATOR': 'opencensus.trace.propagation.google_cloud_format.' 'GoogleCloudFormatPropagator()', - 'BLACKLIST_PATHS': [ + 'EXCLUDELIST_PATHS': [ '_ah/health', ], } diff --git a/tests/system/trace/django/app/urls.py b/tests/system/trace/django/app/urls.py index b111d1894..e9e014cf8 100644 --- a/tests/system/trace/django/app/urls.py +++ b/tests/system/trace/django/app/urls.py @@ -27,14 +27,13 @@ 1. Add an import: from blog import urls as blog_urls 2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls)) """ -from django.conf.urls import include, url +from django.conf.urls import url from django.contrib import admin import app.views - urlpatterns = [ - url(r'^admin/', include(admin.site.urls)), + url(r'^admin/', admin.site.urls), url(r'^$', app.views.home), url(r'^greetings$', app.views.greetings), url(r'^_ah/health$', app.views.health_check), diff --git a/tests/system/trace/django/app/views.py b/tests/system/trace/django/app/views.py index 39873e338..cf1fad4a3 100644 --- a/tests/system/trace/django/app/views.py +++ b/tests/system/trace/django/app/views.py @@ -12,18 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from django.http import HttpResponse -from django.shortcuts import render - -from .forms import HelloForm - -from opencensus.trace import config_integration +import os import mysql.connector import psycopg2 import sqlalchemy +from django.http import HttpResponse +from django.shortcuts import render -import os +from opencensus.trace import config_integration + +from .forms import HelloForm DB_HOST = 'localhost' diff --git a/tests/system/trace/django/django_system_test.py b/tests/system/trace/django/django_system_test.py index e9e01c8cc..c8a101683 100644 --- a/tests/system/trace/django/django_system_test.py +++ b/tests/system/trace/django/django_system_test.py @@ -13,15 +13,14 @@ # limitations under the License. import os -import requests import signal import subprocess +import unittest import uuid +import requests from retrying import retry -import unittest - PROJECT = os.environ.get('GCLOUD_PROJECT_PYTHON') HOST_PORT = 'localhost:8000' @@ -43,7 +42,7 @@ def generate_header(): span_id = uuid.uuid4().hex[:16] trace_option = 1 - header = '{}/{};o={}'.format(trace_id, span_id, trace_option) + header = '{}/{};o={}'.format(trace_id, int(span_id, 16), trace_option) return trace_id, span_id, header @@ -74,7 +73,7 @@ def setUp(self): self.headers_trace = { 'x-cloud-trace-context': - '{}/{};o={}'.format(self.trace_id, self.span_id, 1) + '{}/{};o={}'.format(self.trace_id, int(self.span_id, 16), 1) } # Wait the application to start @@ -134,7 +133,9 @@ def test_with_retry(self): if span.get('name') == '[mysql.query]SELECT 2*3': self.assertEqual( labels.get('mysql.cursor.method.name'), 'execute') - self.assertEqual(labels.get('mysql.query'), 'SELECT 2*3') + self.assertEqual( + labels.get('mysql.query'), 'SELECT 2*3' + ) self.assertTrue(request_succeeded) @@ -168,7 +169,9 @@ def test_with_retry(self): if span.get('name') == '[postgresql.query]SELECT 2*3': self.assertEqual( - labels.get('postgresql.cursor.method.name'), 'execute') + labels.get( + 'postgresql.cursor.method.name'), 'execute' + ) self.assertEqual( labels.get('postgresql.query'), 'SELECT 2*3') diff --git a/tests/system/trace/flask/flask_system_test.py b/tests/system/trace/flask/flask_system_test.py index 440606f1f..7cf97ebb9 100644 --- a/tests/system/trace/flask/flask_system_test.py +++ b/tests/system/trace/flask/flask_system_test.py @@ -13,15 +13,14 @@ # limitations under the License. import os -import requests import signal import subprocess +import unittest import uuid +import requests from retrying import retry -import unittest - PROJECT = os.environ.get('GCLOUD_PROJECT_PYTHON') HOST_PORT = 'localhost:8080' @@ -43,7 +42,7 @@ def generate_header(): span_id = uuid.uuid4().hex[:16] trace_option = 1 - header = '{}/{};o={}'.format(trace_id, span_id, trace_option) + header = '{}/{};o={}'.format(trace_id, int(span_id, 16), trace_option) return trace_id, span_id, header @@ -73,7 +72,7 @@ def setUp(self): self.headers_trace = { 'X-Cloud-Trace-Context': - '{}/{};o={}'.format(self.trace_id, self.span_id, 1) + '{}/{};o={}'.format(self.trace_id, int(self.span_id, 16), 1) } # Wait the application to start @@ -162,7 +161,9 @@ def test_with_retry(self): for span in spans: labels = span.get('labels') if '/http/status_code' in labels.keys(): - self.assertEqual(labels.get('/http/status_code'), '200') + self.assertEqual( + labels.get('/http/status_code'), '200' + ) request_succeeded = True if span.get('name') == '[postgresql.query]SELECT 2*3': diff --git a/tests/unit/common/monitored_resource_util/test_aws_identity_doc_utils.py b/tests/unit/common/monitored_resource_util/test_aws_identity_doc_utils.py index 73f031f45..5658b862e 100644 --- a/tests/unit/common/monitored_resource_util/test_aws_identity_doc_utils.py +++ b/tests/unit/common/monitored_resource_util/test_aws_identity_doc_utils.py @@ -13,9 +13,10 @@ # limitations under the License. import json -import mock import unittest +import mock + from opencensus.common.monitored_resource import aws_identity_doc_utils diff --git a/tests/unit/common/monitored_resource_util/test_gcp_metadata_config.py b/tests/unit/common/monitored_resource_util/test_gcp_metadata_config.py index 10c97776f..d636bcf77 100644 --- a/tests/unit/common/monitored_resource_util/test_gcp_metadata_config.py +++ b/tests/unit/common/monitored_resource_util/test_gcp_metadata_config.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock import os import unittest +import mock + from opencensus.common.monitored_resource import gcp_metadata_config diff --git a/tests/unit/common/monitored_resource_util/test_monitored_resource.py b/tests/unit/common/monitored_resource_util/test_monitored_resource.py index 4cbe537d2..38a1fe4b5 100644 --- a/tests/unit/common/monitored_resource_util/test_monitored_resource.py +++ b/tests/unit/common/monitored_resource_util/test_monitored_resource.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from contextlib import contextmanager import os import sys +from contextlib import contextmanager import mock diff --git a/tests/unit/common/test_http_handler.py b/tests/unit/common/test_http_handler.py index 83f86b66f..cfabbc066 100644 --- a/tests/unit/common/test_http_handler.py +++ b/tests/unit/common/test_http_handler.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest +import json import socket +import unittest + import mock -import json from opencensus.common.http_handler import get_request diff --git a/tests/unit/common/test_schedule.py b/tests/unit/common/test_schedule.py index a6d69b075..4fd85be0a 100644 --- a/tests/unit/common/test_schedule.py +++ b/tests/unit/common/test_schedule.py @@ -14,9 +14,7 @@ import unittest -from opencensus.common.schedule import PeriodicTask -from opencensus.common.schedule import Queue -from opencensus.common.schedule import QueueEvent +from opencensus.common.schedule import PeriodicTask, Queue, QueueEvent TIMEOUT = .1 diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py index 42fc9c57f..49419afa9 100644 --- a/tests/unit/common/test_utils.py +++ b/tests/unit/common/test_utils.py @@ -26,10 +26,11 @@ from opencensus.common.backports import WeakMethod import gc -import mock import unittest import weakref +import mock + from opencensus.common import utils diff --git a/tests/unit/common/transports/test_async.py b/tests/unit/common/transports/test_async.py index 50fce2f5f..ce8e3ab0b 100644 --- a/tests/unit/common/transports/test_async.py +++ b/tests/unit/common/transports/test_async.py @@ -199,7 +199,7 @@ def emit(self, span): # trace2 should be left in the queue because worker is terminated. self.assertEqual(worker._queue.qsize(), 1) - @mock.patch('logging.exception') + @mock.patch('opencensus.common.transports.async_.logger.exception') def test__thread_main_alive_on_emit_failed(self, mock): class Exporter(object): diff --git a/tests/unit/common/transports/test_sync.py b/tests/unit/common/transports/test_sync.py index 71cce563b..c812b025e 100644 --- a/tests/unit/common/transports/test_sync.py +++ b/tests/unit/common/transports/test_sync.py @@ -13,7 +13,9 @@ # limitations under the License. import unittest + import mock + from opencensus.common.transports import sync diff --git a/tests/unit/log/test_log.py b/tests/unit/log/test_log.py index b1cde14eb..528d9e7bd 100644 --- a/tests/unit/log/test_log.py +++ b/tests/unit/log/test_log.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from contextlib import contextmanager import logging import sys +from contextlib import contextmanager import mock diff --git a/tests/unit/metrics/export/test_cumulative.py b/tests/unit/metrics/export/test_cumulative.py index a11b67c45..0a7dc4ebb 100644 --- a/tests/unit/metrics/export/test_cumulative.py +++ b/tests/unit/metrics/export/test_cumulative.py @@ -16,9 +16,7 @@ from mock import Mock -from opencensus.metrics.export import cumulative -from opencensus.metrics.export import gauge -from opencensus.metrics.export import metric_descriptor +from opencensus.metrics.export import cumulative, gauge, metric_descriptor from opencensus.metrics.export import value as value_module diff --git a/tests/unit/metrics/export/test_gauge.py b/tests/unit/metrics/export/test_gauge.py index 2fa1a6afe..bd7ad791c 100644 --- a/tests/unit/metrics/export/test_gauge.py +++ b/tests/unit/metrics/export/test_gauge.py @@ -17,8 +17,7 @@ from mock import Mock -from opencensus.metrics.export import gauge -from opencensus.metrics.export import metric_descriptor +from opencensus.metrics.export import gauge, metric_descriptor from opencensus.metrics.export import value as value_module @@ -394,6 +393,15 @@ def test_create_time_series(self): unused_mock_fn.assert_not_called() self.assertEqual(len(derived_gauge.points.keys()), 1) + # with kwargs + def fn_with_args(value=None): + if value: + return value + return 0 + label_values2 = [1, 2] + point3 = derived_gauge.create_time_series(label_values2, fn_with_args, value=5) # noqa: E501 + self.assertEqual(point3.get_value(), 5) + def test_create_default_time_series(self): derived_gauge = gauge.DerivedLongGauge( Mock(), Mock(), Mock(), [Mock(), Mock]) diff --git a/tests/unit/metrics/export/test_metric.py b/tests/unit/metrics/export/test_metric.py index 610f08a22..730ff1792 100644 --- a/tests/unit/metrics/export/test_metric.py +++ b/tests/unit/metrics/export/test_metric.py @@ -19,9 +19,7 @@ import unittest -from opencensus.metrics.export import metric -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import time_series +from opencensus.metrics.export import metric, metric_descriptor, time_series class TestMetric(unittest.TestCase): diff --git a/tests/unit/metrics/export/test_metric_descriptor.py b/tests/unit/metrics/export/test_metric_descriptor.py index 2a5f7b80c..ff79a3d3c 100644 --- a/tests/unit/metrics/export/test_metric_descriptor.py +++ b/tests/unit/metrics/export/test_metric_descriptor.py @@ -17,8 +17,7 @@ import unittest from opencensus.metrics import label_key -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import value +from opencensus.metrics.export import metric_descriptor, value NAME = 'metric' DESCRIPTION = 'Metric description' diff --git a/tests/unit/metrics/export/test_point.py b/tests/unit/metrics/export/test_point.py index 637c02201..76a937d4f 100644 --- a/tests/unit/metrics/export/test_point.py +++ b/tests/unit/metrics/export/test_point.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest + from opencensus.metrics.export import point as point_module from opencensus.metrics.export import summary as summary_module from opencensus.metrics.export import value as value_module diff --git a/tests/unit/metrics/export/test_summary.py b/tests/unit/metrics/export/test_summary.py index cd7984c58..82c2bdedc 100644 --- a/tests/unit/metrics/export/test_summary.py +++ b/tests/unit/metrics/export/test_summary.py @@ -13,6 +13,7 @@ # limitations under the License. from six import assertRaisesRegex + import unittest from opencensus.metrics.export import summary as summary_module diff --git a/tests/unit/metrics/export/test_time_series.py b/tests/unit/metrics/export/test_time_series.py index b3948361a..57c0ef4aa 100644 --- a/tests/unit/metrics/export/test_time_series.py +++ b/tests/unit/metrics/export/test_time_series.py @@ -17,9 +17,7 @@ import unittest from opencensus.metrics import label_value -from opencensus.metrics.export import point -from opencensus.metrics.export import time_series -from opencensus.metrics.export import value +from opencensus.metrics.export import point, time_series, value START_TIMESTAMP = '2018-10-09T22:33:44.012345Z' LABEL_VALUE1 = label_value.LabelValue('value one') diff --git a/tests/unit/metrics/test_label_key.py b/tests/unit/metrics/test_label_key.py index 5a685e2e2..80c63cace 100644 --- a/tests/unit/metrics/test_label_key.py +++ b/tests/unit/metrics/test_label_key.py @@ -15,6 +15,7 @@ # limitations under the License. import unittest + from opencensus.metrics import label_key as label_key_module diff --git a/tests/unit/metrics/test_label_value.py b/tests/unit/metrics/test_label_value.py index 7af9f2d19..8fe463cd2 100644 --- a/tests/unit/metrics/test_label_value.py +++ b/tests/unit/metrics/test_label_value.py @@ -15,6 +15,7 @@ # limitations under the License. import unittest + from opencensus.metrics import label_value as label_value_module diff --git a/tests/unit/metrics/test_transport.py b/tests/unit/metrics/test_transport.py index 3c6fc36b1..cd2ad0a52 100644 --- a/tests/unit/metrics/test_transport.py +++ b/tests/unit/metrics/test_transport.py @@ -32,24 +32,24 @@ INTERVAL = .1 -class TestMetricExporterTask(unittest.TestCase): +class TestPeriodicMetricTask(unittest.TestCase): def test_default_constructor(self): mock_func = mock.Mock() - task = transport.MetricExporterTask(function=mock_func) + task = transport.PeriodicMetricTask(function=mock_func) self.assertEqual(task.func, mock_func) self.assertEqual(task.interval, transport.DEFAULT_INTERVAL) def test_periodic_task_not_started(self): mock_func = mock.Mock() - task = transport.MetricExporterTask(INTERVAL, mock_func) + task = transport.PeriodicMetricTask(INTERVAL, mock_func) time.sleep(INTERVAL + INTERVAL / 2.0) mock_func.assert_not_called() task.cancel() def test_periodic_task(self): mock_func = mock.Mock() - task = transport.MetricExporterTask(INTERVAL, mock_func) + task = transport.PeriodicMetricTask(INTERVAL, mock_func) task.start() mock_func.assert_not_called() time.sleep(INTERVAL + INTERVAL / 2.0) @@ -58,10 +58,11 @@ def test_periodic_task(self): self.assertEqual(mock_func.call_count, 2) time.sleep(INTERVAL) self.assertEqual(mock_func.call_count, 3) + task.cancel() def test_periodic_task_cancel(self): mock_func = mock.Mock() - task = transport.MetricExporterTask(INTERVAL, mock_func) + task = transport.PeriodicMetricTask(INTERVAL, mock_func) task.start() time.sleep(INTERVAL + INTERVAL / 2.0) self.assertEqual(mock_func.call_count, 1) @@ -69,18 +70,28 @@ def test_periodic_task_cancel(self): time.sleep(INTERVAL) self.assertEqual(mock_func.call_count, 1) + def test_periodic_task_close(self): + mock_func = mock.Mock() + task = transport.PeriodicMetricTask(100, mock_func) + task.start() + mock_func.assert_not_called() + task.close() + self.assertEqual(mock_func.call_count, 1) + @mock.patch('opencensus.metrics.transport.DEFAULT_INTERVAL', INTERVAL) @mock.patch('opencensus.metrics.transport.logger') class TestGetExporterThreadPeriodic(unittest.TestCase): - def test_threaded_export(self, mock_logger): + @mock.patch('opencensus.metrics.transport.itertools.chain') + def test_threaded_export(self, iter_mock, mock_logger): producer = mock.Mock() exporter = mock.Mock() metrics = mock.Mock() producer.get_metrics.return_value = metrics + iter_mock.return_value = producer.get_metrics.return_value try: - task = transport.get_exporter_thread(producer, exporter) + task = transport.get_exporter_thread([producer], exporter) producer.get_metrics.assert_not_called() exporter.export_metrics.assert_not_called() time.sleep(INTERVAL + INTERVAL / 2.0) @@ -96,7 +107,7 @@ def test_producer_error(self, mock_logger): producer.get_metrics.side_effect = ValueError() - task = transport.get_exporter_thread(producer, exporter) + task = transport.get_exporter_thread([producer], exporter) time.sleep(INTERVAL + INTERVAL / 2.0) mock_logger.exception.assert_called() self.assertFalse(task.finished.is_set()) @@ -104,7 +115,7 @@ def test_producer_error(self, mock_logger): def test_producer_deleted(self, mock_logger): producer = mock.Mock() exporter = mock.Mock() - task = transport.get_exporter_thread(producer, exporter) + task = transport.get_exporter_thread([producer], exporter) del producer gc.collect() time.sleep(INTERVAL + INTERVAL / 2.0) @@ -114,9 +125,32 @@ def test_producer_deleted(self, mock_logger): def test_exporter_deleted(self, mock_logger): producer = mock.Mock() exporter = mock.Mock() - task = transport.get_exporter_thread(producer, exporter) + task = transport.get_exporter_thread([producer], exporter) del exporter gc.collect() time.sleep(INTERVAL + INTERVAL / 2.0) mock_logger.exception.assert_called() self.assertTrue(task.finished.is_set()) + + @mock.patch('opencensus.metrics.transport.itertools.chain') + def test_multiple_producers(self, iter_mock, mock_logger): + producer1 = mock.Mock() + producer2 = mock.Mock() + producers = [producer1, producer2] + exporter = mock.Mock() + metrics = mock.Mock() + producer1.get_metrics.return_value = metrics + producer2.get_metrics.return_value = metrics + iter_mock.return_value = metrics + try: + task = transport.get_exporter_thread(producers, exporter) + producer1.get_metrics.assert_not_called() + producer2.get_metrics.assert_not_called() + exporter.export_metrics.assert_not_called() + time.sleep(INTERVAL + INTERVAL / 2.0) + producer1.get_metrics.assert_called_once_with() + producer2.get_metrics.assert_called_once_with() + exporter.export_metrics.assert_called_once_with(metrics) + finally: + task.cancel() + task.join() diff --git a/tests/unit/stats/test_aggregation.py b/tests/unit/stats/test_aggregation.py index 8f56b38fd..671c1910d 100644 --- a/tests/unit/stats/test_aggregation.py +++ b/tests/unit/stats/test_aggregation.py @@ -14,101 +14,96 @@ import unittest -from opencensus.stats import aggregation as aggregation_module - - -class TestBaseAggregation(unittest.TestCase): - def test_constructor_defaults(self): - base_aggregation = aggregation_module.BaseAggregation() - - self.assertEqual(aggregation_module.Type.NONE, - base_aggregation.aggregation_type) - self.assertEqual([], base_aggregation.buckets) - - def test_constructor_explicit(self): - - buckets = ["test"] - base_aggregation = aggregation_module.BaseAggregation(buckets=buckets) +import mock - self.assertEqual(aggregation_module.Type.NONE, - base_aggregation.aggregation_type) - self.assertEqual(["test"], base_aggregation.buckets) +from opencensus.metrics.export import value +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import measure as measure_module class TestSumAggregation(unittest.TestCase): - def test_constructor_defaults(self): + def test_new_aggregation_data_defaults(self): + measure = mock.Mock(spec=measure_module.MeasureInt) sum_aggregation = aggregation_module.SumAggregation() + agg_data = sum_aggregation.new_aggregation_data(measure) + self.assertEqual(0, agg_data.sum_data) + self.assertEqual(value.ValueLong, agg_data.value_type) + + def test_new_aggregation_data_explicit(self): + measure = mock.Mock(spec=measure_module.MeasureInt) + sum_aggregation = aggregation_module.SumAggregation(sum=1) + agg_data = sum_aggregation.new_aggregation_data(measure) + self.assertEqual(1, agg_data.sum_data) + self.assertEqual(value.ValueLong, agg_data.value_type) + + def test_new_aggregation_data_float(self): + measure = mock.Mock(spec=measure_module.MeasureFloat) + sum_aggregation = aggregation_module.SumAggregation() + agg_data = sum_aggregation.new_aggregation_data(measure) + self.assertEqual(0, agg_data.sum_data) + self.assertEqual(value.ValueDouble, agg_data.value_type) - self.assertEqual(0, sum_aggregation.sum.sum_data) - self.assertEqual(aggregation_module.Type.SUM, - sum_aggregation.aggregation_type) - - def test_constructor_explicit(self): - sum = 1 - - sum_aggregation = aggregation_module.SumAggregation(sum=sum) - - self.assertEqual(1, sum_aggregation.sum.sum_data) - self.assertEqual(aggregation_module.Type.SUM, - sum_aggregation.aggregation_type) + def test_new_aggregation_data_bad(self): + measure = mock.Mock(spec=measure_module.BaseMeasure) + sum_aggregation = aggregation_module.SumAggregation() + with self.assertRaises(ValueError): + sum_aggregation.new_aggregation_data(measure) class TestCountAggregation(unittest.TestCase): - def test_constructor_defaults(self): + def test_new_aggregation_data_defaults(self): count_aggregation = aggregation_module.CountAggregation() + agg_data = count_aggregation.new_aggregation_data() + self.assertEqual(0, agg_data.count_data) - self.assertEqual(0, count_aggregation.count.count_data) - self.assertEqual(aggregation_module.Type.COUNT, - count_aggregation.aggregation_type) - - def test_constructor_explicit(self): - count = 4 - - count_aggregation = aggregation_module.CountAggregation(count=count) - - self.assertEqual(4, count_aggregation.count.count_data) - self.assertEqual(aggregation_module.Type.COUNT, - count_aggregation.aggregation_type) + def test_new_aggregation_data_explicit(self): + count_aggregation = aggregation_module.CountAggregation(count=4) + agg_data = count_aggregation.new_aggregation_data() + self.assertEqual(4, agg_data.count_data) class TestLastValueAggregation(unittest.TestCase): - def test_constructor_defaults(self): + def test_new_aggregation_data_defaults(self): + measure = mock.Mock(spec=measure_module.MeasureInt) last_value_aggregation = aggregation_module.LastValueAggregation() + agg_data = last_value_aggregation.new_aggregation_data(measure) + self.assertEqual(0, agg_data.value) + self.assertEqual(value.ValueLong, agg_data.value_type) - self.assertEqual(0, last_value_aggregation.value) - self.assertEqual(aggregation_module.Type.LASTVALUE, - last_value_aggregation.aggregation_type) - - def test_constructor_explicit(self): - val = 16 + def test_new_aggregation_data_explicit(self): + measure = mock.Mock(spec=measure_module.MeasureInt) last_value_aggregation = aggregation_module.LastValueAggregation( - value=val) + value=6) + agg_data = last_value_aggregation.new_aggregation_data(measure) + self.assertEqual(6, agg_data.value) + self.assertEqual(value.ValueLong, agg_data.value_type) - self.assertEqual(16, last_value_aggregation.value) - self.assertEqual(aggregation_module.Type.LASTVALUE, - last_value_aggregation.aggregation_type) + def test_new_aggregation_data_float(self): + measure = mock.Mock(spec=measure_module.MeasureFloat) + last_value_aggregation = aggregation_module.LastValueAggregation() + agg_data = last_value_aggregation.new_aggregation_data(measure) + self.assertEqual(0, agg_data.value) + self.assertEqual(value.ValueDouble, agg_data.value_type) + + def test_new_aggregation_data_bad(self): + measure = mock.Mock(spec=measure_module.BaseMeasure) + last_value_aggregation = aggregation_module.LastValueAggregation() + with self.assertRaises(ValueError): + last_value_aggregation.new_aggregation_data(measure) class TestDistributionAggregation(unittest.TestCase): - def test_constructor_defaults(self): + def test_new_aggregation_data_defaults(self): distribution_aggregation = aggregation_module.DistributionAggregation() + agg_data = distribution_aggregation.new_aggregation_data() + self.assertEqual([], agg_data.bounds) - self.assertEqual([], distribution_aggregation.boundaries.boundaries) - self.assertEqual({}, distribution_aggregation.distribution) - self.assertEqual(aggregation_module.Type.DISTRIBUTION, - distribution_aggregation.aggregation_type) - - def test_constructor_explicit(self): + def test_new_aggregation_data_explicit(self): boundaries = [1, 2] - distribution = [0, 1, 2] distribution_aggregation = aggregation_module.DistributionAggregation( - boundaries=boundaries, distribution=distribution) - - self.assertEqual([1, 2], - distribution_aggregation.boundaries.boundaries) - self.assertEqual([0, 1, 2], distribution_aggregation.distribution) - self.assertEqual(aggregation_module.Type.DISTRIBUTION, - distribution_aggregation.aggregation_type) + boundaries=boundaries) + agg_data = distribution_aggregation.new_aggregation_data() + self.assertEqual(boundaries, agg_data.bounds) def test_init_bad_boundaries(self): """Check that boundaries must be sorted and unique.""" @@ -120,9 +115,7 @@ def test_init_bad_boundaries(self): def test_init_negative_boundaries(self): """Check that non-positive boundaries are dropped.""" da = aggregation_module.DistributionAggregation([-2, -1, 0, 1, 2]) - self.assertEqual(da.boundaries.boundaries, [1, 2]) - self.assertEqual(da.aggregation_data.bounds, [1, 2]) + self.assertEqual(da.new_aggregation_data().bounds, [1, 2]) da2 = aggregation_module.DistributionAggregation([-2, -1]) - self.assertEqual(da2.boundaries.boundaries, []) - self.assertEqual(da2.aggregation_data.bounds, []) + self.assertEqual(da2.new_aggregation_data().bounds, []) diff --git a/tests/unit/stats/test_aggregation_data.py b/tests/unit/stats/test_aggregation_data.py index 95b94fb1e..d1870f2c9 100644 --- a/tests/unit/stats/test_aggregation_data.py +++ b/tests/unit/stats/test_aggregation_data.py @@ -12,104 +12,131 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime import time import unittest +from datetime import datetime import mock from opencensus.metrics.export import point -from opencensus.metrics.export import value +from opencensus.metrics.export import value as value_module from opencensus.stats import aggregation_data as aggregation_data_module -class TestBaseAggregationData(unittest.TestCase): - def test_constructor(self): - aggregation_data = 0 - base_aggregation_data = aggregation_data_module.BaseAggregationData( - aggregation_data=aggregation_data) - - self.assertEqual(0, base_aggregation_data.aggregation_data) +class TestSumAggregationData(unittest.TestCase): + def test_constructor_float(self): + sum_data = 1.0 + sum_aggregation_data = aggregation_data_module.SumAggregationData( + value_type=value_module.ValueDouble, sum_data=sum_data) + self.assertEqual(1.0, sum_aggregation_data.sum_data) -class TestSumAggregationData(unittest.TestCase): - def test_constructor(self): + def test_constructor_int(self): sum_data = 1 - sum_aggregation_data = aggregation_data_module.SumAggregationDataFloat( - sum_data=sum_data) + sum_aggregation_data = aggregation_data_module.SumAggregationData( + value_type=value_module.ValueLong, sum_data=sum_data) self.assertEqual(1, sum_aggregation_data.sum_data) - def test_add_sample(self): + def test_add_sample_float(self): + sum_data = 1 + value = 3.5 + sum_aggregation_data = aggregation_data_module.SumAggregationData( + value_type=value_module.ValueDouble, sum_data=sum_data) + sum_aggregation_data.add_sample(value, None, None) + + self.assertEqual(4.5, sum_aggregation_data.sum_data) + + def test_add_sample_int(self): sum_data = 1 value = 3 - sum_aggregation_data = aggregation_data_module.SumAggregationDataFloat( - sum_data=sum_data) + sum_aggregation_data = aggregation_data_module.SumAggregationData( + value_type=value_module.ValueLong, sum_data=sum_data) sum_aggregation_data.add_sample(value, None, None) self.assertEqual(4, sum_aggregation_data.sum_data) - def test_to_point(self): + def test_to_point_float(self): sum_data = 12.345 timestamp = datetime(1970, 1, 1) - agg = aggregation_data_module.SumAggregationDataFloat(sum_data) + agg = aggregation_data_module.SumAggregationData( + value_type=value_module.ValueDouble, sum_data=sum_data) converted_point = agg.to_point(timestamp) self.assertTrue(isinstance(converted_point, point.Point)) - self.assertTrue(isinstance(converted_point.value, value.ValueDouble)) + self.assertTrue(isinstance(converted_point.value, + value_module.ValueDouble)) self.assertEqual(converted_point.value.value, sum_data) self.assertEqual(converted_point.timestamp, timestamp) - -class TestCountAggregationData(unittest.TestCase): - def test_constructor(self): - count_data = 0 - count_aggregation_data = aggregation_data_module.CountAggregationData( - count_data=count_data) - - self.assertEqual(0, count_aggregation_data.count_data) - - def test_add_sample(self): - count_data = 0 - count_aggregation_data = aggregation_data_module.CountAggregationData( - count_data=count_data) - count_aggregation_data.add_sample(10, None, None) - - self.assertEqual(1, count_aggregation_data.count_data) - - def test_to_point(self): - count_data = 123 + def test_to_point_int(self): + sum_data = 12 timestamp = datetime(1970, 1, 1) - agg = aggregation_data_module.CountAggregationData(count_data) + agg = aggregation_data_module.SumAggregationData( + value_type=value_module.ValueLong, sum_data=sum_data) converted_point = agg.to_point(timestamp) self.assertTrue(isinstance(converted_point, point.Point)) - self.assertTrue(isinstance(converted_point.value, value.ValueLong)) - self.assertEqual(converted_point.value.value, count_data) + self.assertTrue(isinstance(converted_point.value, + value_module.ValueLong)) + self.assertEqual(converted_point.value.value, sum_data) self.assertEqual(converted_point.timestamp, timestamp) class TestLastValueAggregationData(unittest.TestCase): - def test_constructor(self): + def test_constructor_float(self): + value_data = 0.0 + last_value_aggregation_data =\ + aggregation_data_module.LastValueAggregationData( + value_type=value_module.ValueDouble, value=value_data) + + self.assertEqual(0.0, last_value_aggregation_data.value) + + def test_constructor_int(self): value_data = 0 last_value_aggregation_data =\ - aggregation_data_module.LastValueAggregationData(value=value_data) + aggregation_data_module.LastValueAggregationData( + value_type=value_module.ValueLong, value=value_data) self.assertEqual(0, last_value_aggregation_data.value) - def test_overwrite_sample(self): + def test_overwrite_sample_float(self): first_data = 0 last_value_aggregation_data =\ - aggregation_data_module.LastValueAggregationData(value=first_data) + aggregation_data_module.LastValueAggregationData( + value_type=value_module.ValueDouble, value=first_data) + self.assertEqual(0, last_value_aggregation_data.value) + last_value_aggregation_data.add_sample(1.2, None, None) + self.assertEqual(1.2, last_value_aggregation_data.value) + + def test_overwrite_sample_int(self): + first_data = 0 + last_value_aggregation_data =\ + aggregation_data_module.LastValueAggregationData( + value_type=value_module.ValueLong, value=first_data) self.assertEqual(0, last_value_aggregation_data.value) last_value_aggregation_data.add_sample(1, None, None) self.assertEqual(1, last_value_aggregation_data.value) - def test_to_point(self): + def test_to_point_float(self): val = 1.2 timestamp = datetime(1970, 1, 1) - agg = aggregation_data_module.LastValueAggregationData(val) + agg = aggregation_data_module.LastValueAggregationData( + value_type=value_module.ValueDouble, value=val) + converted_point = agg.to_point(timestamp) + self.assertTrue(isinstance(converted_point, point.Point)) + self.assertTrue(isinstance(converted_point.value, + value_module.ValueDouble)) + self.assertEqual(converted_point.value.value, val) + self.assertEqual(converted_point.timestamp, timestamp) + + def test_to_pointInt(self): + val = 1 + timestamp = datetime(1970, 1, 1) + agg = aggregation_data_module.LastValueAggregationData( + value_type=value_module.ValueLong, value=val) converted_point = agg.to_point(timestamp) self.assertTrue(isinstance(converted_point, point.Point)) - self.assertTrue(isinstance(converted_point.value, value.ValueDouble)) + self.assertTrue(isinstance(converted_point.value, + value_module.ValueLong)) self.assertEqual(converted_point.value.value, val) self.assertEqual(converted_point.timestamp, timestamp) @@ -117,7 +144,7 @@ def test_to_point(self): def exemplars_equal(stats_ex, metrics_ex): """Compare a stats exemplar to a metrics exemplar.""" assert isinstance(stats_ex, aggregation_data_module.Exemplar) - assert isinstance(metrics_ex, value.Exemplar) + assert isinstance(metrics_ex, value_module.Exemplar) return (stats_ex.value == metrics_ex.value and stats_ex.timestamp == metrics_ex.timestamp and stats_ex.attachments == metrics_ex.attachments) @@ -474,7 +501,7 @@ def test_to_point(self): ) converted_point = dist_agg_data.to_point(timestamp) self.assertTrue(isinstance(converted_point.value, - value.ValueDistribution)) + value_module.ValueDistribution)) self.assertEqual(converted_point.value.count, 99) self.assertEqual(converted_point.value.sum, 4950) self.assertEqual(converted_point.value.sum_of_squared_deviation, @@ -501,7 +528,7 @@ def test_to_point_no_histogram(self): ) converted_point = dist_agg_data.to_point(timestamp) self.assertTrue(isinstance(converted_point.value, - value.ValueDistribution)) + value_module.ValueDistribution)) self.assertEqual(converted_point.value.count, 99) self.assertEqual(converted_point.value.sum, 4950) self.assertEqual(converted_point.value.sum_of_squared_deviation, diff --git a/tests/unit/stats/test_base_stats.py b/tests/unit/stats/test_base_stats.py index 78723f36b..1d403c7dc 100644 --- a/tests/unit/stats/test_base_stats.py +++ b/tests/unit/stats/test_base_stats.py @@ -13,7 +13,9 @@ # limitations under the License. import unittest + import mock + from opencensus.stats import base_exporter diff --git a/tests/unit/stats/test_measure_to_view_map.py b/tests/unit/stats/test_measure_to_view_map.py index 4beb0cb98..a754baf04 100644 --- a/tests/unit/stats/test_measure_to_view_map.py +++ b/tests/unit/stats/test_measure_to_view_map.py @@ -18,13 +18,11 @@ from opencensus.stats import measure_to_view_map as measure_to_view_map_module from opencensus.stats.aggregation import CountAggregation -from opencensus.stats.measure import BaseMeasure -from opencensus.stats.measure import MeasureInt +from opencensus.stats.measure import BaseMeasure, MeasureInt from opencensus.stats.view import View from opencensus.stats.view_data import ViewData from opencensus.tags import tag_key as tag_key_module - METHOD_KEY = tag_key_module.TagKey("method") REQUEST_COUNT_MEASURE = MeasureInt( "request_count", "number of requests", "1") diff --git a/tests/unit/stats/test_measurement.py b/tests/unit/stats/test_measurement.py index 431c96863..3780a8657 100644 --- a/tests/unit/stats/test_measurement.py +++ b/tests/unit/stats/test_measurement.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opencensus.stats import measurement as measurement_module - import unittest +from opencensus.stats import measurement as measurement_module + class TestMeasurement(unittest.TestCase): def test_constructor(self): diff --git a/tests/unit/stats/test_measurement_map.py b/tests/unit/stats/test_measurement_map.py index ee103771f..89add276b 100644 --- a/tests/unit/stats/test_measurement_map.py +++ b/tests/unit/stats/test_measurement_map.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock import unittest +import mock + from opencensus.stats import measurement_map as measurement_map_module -from opencensus.tags import Tag -from opencensus.tags import TagContext -from opencensus.tags import TagMap +from opencensus.tags import Tag, TagContext, TagMap logger_patch = mock.patch('opencensus.stats.measurement_map.logger') diff --git a/tests/unit/stats/test_metric_utils.py b/tests/unit/stats/test_metric_utils.py index b51b03da8..355f0da2e 100644 --- a/tests/unit/stats/test_metric_utils.py +++ b/tests/unit/stats/test_metric_utils.py @@ -17,77 +17,20 @@ import mock -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import point -from opencensus.metrics.export import value -from opencensus.stats import aggregation -from opencensus.stats import aggregation_data -from opencensus.stats import measure -from opencensus.stats import metric_utils -from opencensus.stats import view -from opencensus.stats import view_data -from opencensus.tags import tag_key -from opencensus.tags import tag_value +from opencensus.metrics.export import metric_descriptor, point, value +from opencensus.stats import ( + aggregation, + aggregation_data, + measure, + metric_utils, + view, + view_data, +) +from opencensus.tags import tag_key, tag_value class TestMetricUtils(unittest.TestCase): - def test_get_metric_type(self): - measure_int = mock.Mock(spec=measure.MeasureInt) - measure_float = mock.Mock(spec=measure.MeasureFloat) - agg_sum = mock.Mock(spec=aggregation.SumAggregation) - agg_sum.aggregation_type = aggregation.Type.SUM - agg_count = mock.Mock(spec=aggregation.CountAggregation) - agg_count.aggregation_type = aggregation.Type.COUNT - agg_dist = mock.Mock(spec=aggregation.DistributionAggregation) - agg_dist.aggregation_type = aggregation.Type.DISTRIBUTION - agg_lv = mock.Mock(spec=aggregation.LastValueAggregation) - agg_lv.aggregation_type = aggregation.Type.LASTVALUE - - view_to_metric_type = { - (measure_int, agg_sum): - metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64, - (measure_int, agg_count): - metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64, - (measure_int, agg_dist): - metric_descriptor.MetricDescriptorType.CUMULATIVE_DISTRIBUTION, - (measure_int, agg_lv): - metric_descriptor.MetricDescriptorType.GAUGE_INT64, - (measure_float, agg_sum): - metric_descriptor.MetricDescriptorType.CUMULATIVE_DOUBLE, - (measure_float, agg_count): - metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64, - (measure_float, agg_dist): - metric_descriptor.MetricDescriptorType.CUMULATIVE_DISTRIBUTION, - (measure_float, agg_lv): - metric_descriptor.MetricDescriptorType.GAUGE_DOUBLE, - } - - for (mm, ma), metric_type in view_to_metric_type.items(): - self.assertEqual(metric_utils.get_metric_type(mm, ma), metric_type) - - def test_get_metric_type_bad_aggregation(self): - base_agg = mock.Mock(spec=aggregation.BaseAggregation) - base_agg.aggregation_type = aggregation.Type.NONE - with self.assertRaises(ValueError): - metric_utils.get_metric_type(mock.Mock(), base_agg) - - bad_agg = mock.Mock(spec=aggregation.SumAggregation) - bad_agg.aggregation_type = aggregation.Type.COUNT - with self.assertRaises(AssertionError): - metric_utils.get_metric_type(mock.Mock(), bad_agg) - - def test_get_metric_type_bad_measure(self): - base_measure = mock.Mock(spec=measure.BaseMeasure) - agg_sum = mock.Mock(spec=aggregation.SumAggregation) - agg_sum.aggregation_type = aggregation.Type.SUM - agg_lv = mock.Mock(spec=aggregation.LastValueAggregation) - agg_lv.aggregation_type = aggregation.Type.LASTVALUE - with self.assertRaises(ValueError): - metric_utils.get_metric_type(base_measure, agg_sum) - with self.assertRaises(ValueError): - metric_utils.get_metric_type(base_measure, agg_lv) - - def do_test_view_data_to_metric(self, aggregation_type, aggregation_class, + def do_test_view_data_to_metric(self, aggregation_class, value_type, metric_descriptor_type): """Test that ViewDatas are converted correctly into Metrics. @@ -101,7 +44,7 @@ def do_test_view_data_to_metric(self, aggregation_type, aggregation_class, mock_measure = mock.Mock(spec=measure.MeasureFloat) mock_aggregation = mock.Mock(spec=aggregation_class) - mock_aggregation.aggregation_type = aggregation_type + mock_aggregation.get_metric_type.return_value = metric_descriptor_type vv = view.View( name=mock.Mock(), @@ -117,7 +60,7 @@ def do_test_view_data_to_metric(self, aggregation_type, aggregation_class, mock_point = mock.Mock(spec=point.Point) mock_point.value = mock.Mock(spec=value_type) - mock_agg = mock.Mock(spec=aggregation_data.SumAggregationDataFloat) + mock_agg = mock.Mock(spec=aggregation_data.SumAggregationData) mock_agg.to_point.return_value = mock_point vd.tag_value_aggregation_data_map = { @@ -148,19 +91,16 @@ def do_test_view_data_to_metric(self, aggregation_type, aggregation_class, def test_view_data_to_metric(self): args_list = [ [ - aggregation.Type.SUM, aggregation.SumAggregation, value.ValueDouble, metric_descriptor.MetricDescriptorType.CUMULATIVE_DOUBLE ], [ - aggregation.Type.COUNT, aggregation.CountAggregation, value.ValueLong, metric_descriptor.MetricDescriptorType.CUMULATIVE_INT64 ], [ - aggregation.Type.DISTRIBUTION, aggregation.DistributionAggregation, value.ValueDistribution, metric_descriptor.MetricDescriptorType.CUMULATIVE_DISTRIBUTION @@ -172,7 +112,8 @@ def test_view_data_to_metric(self): def test_convert_view_without_labels(self): mock_measure = mock.Mock(spec=measure.MeasureFloat) mock_aggregation = mock.Mock(spec=aggregation.DistributionAggregation) - mock_aggregation.aggregation_type = aggregation.Type.DISTRIBUTION + mock_aggregation.get_metric_type.return_value = \ + metric_descriptor.MetricDescriptorType.CUMULATIVE_DISTRIBUTION vd = mock.Mock(spec=view_data.ViewData) vd.view = view.View( @@ -186,7 +127,7 @@ def test_convert_view_without_labels(self): mock_point = mock.Mock(spec=point.Point) mock_point.value = mock.Mock(spec=value.ValueDistribution) - mock_agg = mock.Mock(spec=aggregation_data.SumAggregationDataFloat) + mock_agg = mock.Mock(spec=aggregation_data.DistributionAggregationData) mock_agg.to_point.return_value = mock_point vd.tag_value_aggregation_data_map = {tuple(): mock_agg} diff --git a/tests/unit/stats/test_stats.py b/tests/unit/stats/test_stats.py index 81588f41b..e120ac499 100644 --- a/tests/unit/stats/test_stats.py +++ b/tests/unit/stats/test_stats.py @@ -19,10 +19,8 @@ import unittest -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import value -from opencensus.stats import aggregation -from opencensus.stats import measure +from opencensus.metrics.export import metric_descriptor, value +from opencensus.stats import aggregation, measure from opencensus.stats import stats as stats_module from opencensus.stats import view from opencensus.tags import tag_map @@ -59,6 +57,8 @@ def test_get_metrics(self): mm._measurement_map = {mock_measure: 1.0} mock_view.aggregation = aggregation.DistributionAggregation() + mock_view.new_aggregation_data.return_value = \ + mock_view.aggregation.new_aggregation_data() tm = tag_map.TagMap() tm.insert('k1', 'v1') diff --git a/tests/unit/stats/test_stats_recorder.py b/tests/unit/stats/test_stats_recorder.py index e23b1ae82..ce22597e9 100644 --- a/tests/unit/stats/test_stats_recorder.py +++ b/tests/unit/stats/test_stats_recorder.py @@ -13,10 +13,12 @@ # limitations under the License. import unittest + import mock + +from opencensus.stats import execution_context from opencensus.stats import stats_recorder as stats_recorder_module from opencensus.stats.measurement_map import MeasurementMap -from opencensus.stats import execution_context class TestStatsRecorder(unittest.TestCase): diff --git a/tests/unit/stats/test_view.py b/tests/unit/stats/test_view.py index d1f9d3ece..30dca3cdc 100644 --- a/tests/unit/stats/test_view.py +++ b/tests/unit/stats/test_view.py @@ -13,12 +13,11 @@ # limitations under the License. import unittest + import mock from opencensus.metrics.export import metric_descriptor -from opencensus.stats import aggregation -from opencensus.stats import measure -from opencensus.stats import view +from opencensus.stats import aggregation, measure from opencensus.stats import view as view_module @@ -46,9 +45,10 @@ def test_constructor(self): def test_view_to_metric_descriptor(self): mock_measure = mock.Mock(spec=measure.MeasureFloat) mock_agg = mock.Mock(spec=aggregation.SumAggregation) - mock_agg.aggregation_type = aggregation.Type.SUM - test_view = view.View("name", "description", ["tk1", "tk2"], - mock_measure, mock_agg) + mock_agg.get_metric_type.return_value = \ + metric_descriptor.MetricDescriptorType.CUMULATIVE_DOUBLE + test_view = view_module.View("name", "description", ["tk1", "tk2"], + mock_measure, mock_agg) self.assertIsNone(test_view._metric_descriptor) md = test_view.get_metric_descriptor() diff --git a/tests/unit/stats/test_view_data.py b/tests/unit/stats/test_view_data.py index 1edf3008c..87791802a 100644 --- a/tests/unit/stats/test_view_data.py +++ b/tests/unit/stats/test_view_data.py @@ -12,15 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import unittest from datetime import datetime + import mock -import unittest from opencensus.common import utils from opencensus.stats import aggregation as aggregation_module from opencensus.stats import measure as measure_module -from opencensus.stats import view_data as view_data_module from opencensus.stats import view as view_module +from opencensus.stats import view_data as view_data_module class TestViewData(unittest.TestCase): @@ -118,9 +119,8 @@ def test_record(self): def test_record_with_attachment(self): boundaries = [1, 2, 3] - distribution = {1: "test"} distribution_aggregation = aggregation_module.DistributionAggregation( - boundaries=boundaries, distribution=distribution) + boundaries=boundaries) name = "testName" description = "testMeasure" unit = "testUnit" @@ -168,9 +168,8 @@ def test_record_with_attachment(self): def test_record_with_attachment_no_histogram(self): boundaries = None - distribution = {1: "test"} distribution_aggregation = aggregation_module.DistributionAggregation( - boundaries=boundaries, distribution=distribution) + boundaries=boundaries) name = "testName" description = "testMeasure" unit = "testUnit" @@ -215,7 +214,7 @@ def test_record_with_attachment_no_histogram(self): view_data.tag_value_aggregation_data_map[tuple_vals].exemplars) def test_record_with_multi_keys(self): - measure = mock.Mock() + measure = mock.Mock(spec=measure_module.MeasureInt) sum_aggregation = aggregation_module.SumAggregation() view = view_module.View("test_view", "description", ['key1', 'key2'], measure, sum_aggregation) @@ -271,7 +270,7 @@ def test_record_with_multi_keys(self): self.assertEqual(2, sum_data_2.sum_data) def test_record_with_missing_key_in_context(self): - measure = mock.Mock() + measure = mock.Mock(spec=measure_module.MeasureInt) sum_aggregation = aggregation_module.SumAggregation() view = view_module.View("test_view", "description", ['key1', 'key2'], measure, sum_aggregation) @@ -297,7 +296,7 @@ def test_record_with_missing_key_in_context(self): self.assertEqual(4, sum_data.sum_data) def test_record_with_none_context(self): - measure = mock.Mock() + measure = mock.Mock(spec=measure_module.MeasureInt) sum_aggregation = aggregation_module.SumAggregation() view = view_module.View("test_view", "description", ['key1', 'key2'], measure, sum_aggregation) diff --git a/tests/unit/stats/test_view_manager.py b/tests/unit/stats/test_view_manager.py index ded2ecc74..b6dea72bc 100644 --- a/tests/unit/stats/test_view_manager.py +++ b/tests/unit/stats/test_view_manager.py @@ -13,9 +13,11 @@ # limitations under the License. import unittest + import mock -from opencensus.stats import view_manager as view_manager_module + from opencensus.stats import execution_context +from opencensus.stats import view_manager as view_manager_module from opencensus.stats.measure_to_view_map import MeasureToViewMap diff --git a/tests/unit/tags/test_tag.py b/tests/unit/tags/test_tag.py index 21c5d0eb3..6deac8fc5 100644 --- a/tests/unit/tags/test_tag.py +++ b/tests/unit/tags/test_tag.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest + from opencensus.tags import Tag diff --git a/tests/unit/tags/test_tag_value.py b/tests/unit/tags/test_tag_value.py index f5aa0b771..dfd10d565 100644 --- a/tests/unit/tags/test_tag_value.py +++ b/tests/unit/tags/test_tag_value.py @@ -15,6 +15,7 @@ # limitations under the License. import unittest + from opencensus.tags import TagValue diff --git a/tests/unit/trace/exporters/test_logging_exporter.py b/tests/unit/trace/exporters/test_logging_exporter.py index 36ca52795..4dd45e938 100644 --- a/tests/unit/trace/exporters/test_logging_exporter.py +++ b/tests/unit/trace/exporters/test_logging_exporter.py @@ -17,8 +17,7 @@ import mock -from opencensus.trace import logging_exporter -from opencensus.trace import span_context +from opencensus.trace import logging_exporter, span_context from opencensus.trace import span_data as span_data_module diff --git a/tests/unit/trace/propagation/test_b3_format.py b/tests/unit/trace/propagation/test_b3_format.py index 75ce604ed..68feb20f0 100644 --- a/tests/unit/trace/propagation/test_b3_format.py +++ b/tests/unit/trace/propagation/test_b3_format.py @@ -13,10 +13,11 @@ # limitations under the License. import unittest + import mock -from opencensus.trace.span_context import INVALID_SPAN_ID from opencensus.trace.propagation import b3_format +from opencensus.trace.span_context import INVALID_SPAN_ID class TestB3FormatPropagator(unittest.TestCase): diff --git a/tests/unit/trace/propagation/test_google_cloud_format.py b/tests/unit/trace/propagation/test_google_cloud_format.py index f7d753e77..4fe455504 100644 --- a/tests/unit/trace/propagation/test_google_cloud_format.py +++ b/tests/unit/trace/propagation/test_google_cloud_format.py @@ -53,7 +53,7 @@ def test_header_type_error(self): def test_header_match(self): # Trace option is not enabled. - header = '6e0c63257de34c92bf9efcd03927272e/00f067aa0ba902b7;o=0' + header = '6e0c63257de34c92bf9efcd03927272e/67667974448284343;o=0' expected_trace_id = '6e0c63257de34c92bf9efcd03927272e' expected_span_id = '00f067aa0ba902b7' @@ -65,7 +65,7 @@ def test_header_match(self): self.assertFalse(span_context.trace_options.enabled) # Trace option is enabled. - header = '6e0c63257de34c92bf9efcd03927272e/00f067aa0ba902b7;o=1' + header = '6e0c63257de34c92bf9efcd03927272e/67667974448284343;o=1' expected_trace_id = '6e0c63257de34c92bf9efcd03927272e' expected_span_id = '00f067aa0ba902b7' @@ -76,8 +76,58 @@ def test_header_match(self): self.assertEqual(span_context.span_id, expected_span_id) self.assertTrue(span_context.trace_options.enabled) + def test_header_match_no_span_id(self): + # Trace option is not enabled. + header = '6e0c63257de34c92bf9efcd03927272e;o=0' + expected_trace_id = '6e0c63257de34c92bf9efcd03927272e' + expected_span_id = None + + propagator = google_cloud_format.GoogleCloudFormatPropagator() + span_context = propagator.from_header(header) + + self.assertEqual(span_context.trace_id, expected_trace_id) + self.assertEqual(span_context.span_id, expected_span_id) + self.assertFalse(span_context.trace_options.enabled) + + # Trace option is enabled. + header = '6e0c63257de34c92bf9efcd03927272e;o=1' + expected_trace_id = '6e0c63257de34c92bf9efcd03927272e' + expected_span_id = None + + propagator = google_cloud_format.GoogleCloudFormatPropagator() + span_context = propagator.from_header(header) + + self.assertEqual(span_context.trace_id, expected_trace_id) + self.assertEqual(span_context.span_id, expected_span_id) + self.assertTrue(span_context.trace_options.enabled) + + def test_header_match_empty_span_id(self): + # Trace option is not enabled. + header = '6e0c63257de34c92bf9efcd03927272e/;o=0' + expected_trace_id = '6e0c63257de34c92bf9efcd03927272e' + expected_span_id = None + + propagator = google_cloud_format.GoogleCloudFormatPropagator() + span_context = propagator.from_header(header) + + self.assertEqual(span_context.trace_id, expected_trace_id) + self.assertEqual(span_context.span_id, expected_span_id) + self.assertFalse(span_context.trace_options.enabled) + + # Trace option is enabled. + header = '6e0c63257de34c92bf9efcd03927272e/;o=1' + expected_trace_id = '6e0c63257de34c92bf9efcd03927272e' + expected_span_id = None + + propagator = google_cloud_format.GoogleCloudFormatPropagator() + span_context = propagator.from_header(header) + + self.assertEqual(span_context.trace_id, expected_trace_id) + self.assertEqual(span_context.span_id, expected_span_id) + self.assertTrue(span_context.trace_options.enabled) + def test_header_match_no_option(self): - header = '6e0c63257de34c92bf9efcd03927272e/00f067aa0ba902b7' + header = '6e0c63257de34c92bf9efcd03927272e/67667974448284343' expected_trace_id = '6e0c63257de34c92bf9efcd03927272e' expected_span_id = '00f067aa0ba902b7' @@ -101,7 +151,7 @@ def test_headers_match(self): # Trace option is enabled. headers = { 'X-Cloud-Trace-Context': - '6e0c63257de34c92bf9efcd03927272e/00f067aa0ba902b7;o=1', + '6e0c63257de34c92bf9efcd03927272e/67667974448284343;o=1', } expected_trace_id = '6e0c63257de34c92bf9efcd03927272e' expected_span_id = '00f067aa0ba902b7' @@ -128,7 +178,7 @@ def test_to_header(self): header = propagator.to_header(span_context) expected_header = '{}/{};o={}'.format( - trace_id, span_id, 1) + trace_id, int(span_id, 16), 1) self.assertEqual(header, expected_header) @@ -147,7 +197,8 @@ def test_to_headers(self): headers = propagator.to_headers(span_context) expected_headers = { - 'X-Cloud-Trace-Context': '{}/{};o={}'.format(trace_id, span_id, 1), + 'X-Cloud-Trace-Context': '{}/{};o={}'.format( + trace_id, int(span_id, 16), 1), } self.assertEqual(headers, expected_headers) diff --git a/tests/unit/trace/test_base_span.py b/tests/unit/trace/test_base_span.py index d5c6d2eb1..d27bc6826 100644 --- a/tests/unit/trace/test_base_span.py +++ b/tests/unit/trace/test_base_span.py @@ -13,7 +13,9 @@ # limitations under the License. import unittest + import mock + from opencensus.trace.base_span import BaseSpan @@ -73,6 +75,12 @@ def test_add_link_abstract(self): with self.assertRaises(NotImplementedError): span.add_link(None) + def test_set_status_abstract(self): + span = BaseSpan() + + with self.assertRaises(NotImplementedError): + span.set_status(None) + def test_iter_abstract(self): span = BaseSpan() diff --git a/tests/unit/trace/test_blank_span.py b/tests/unit/trace/test_blank_span.py index 82ed3f003..d9d1e1c15 100644 --- a/tests/unit/trace/test_blank_span.py +++ b/tests/unit/trace/test_blank_span.py @@ -13,12 +13,14 @@ # limitations under the License. import datetime -import mock import unittest +import mock + from opencensus.common import utils from opencensus.trace.link import Link from opencensus.trace.span import format_span_json +from opencensus.trace.status import Status from opencensus.trace.time_event import MessageEvent @@ -59,6 +61,9 @@ def test_do_not_crash(self): link = Link(span_id='1234', trace_id='4567') span.add_link(link) + status = Status(0, 'Ok', {'details': 'ok'}) + span.set_status(status) + message_event = mock.Mock() message_event = MessageEvent(datetime.datetime.utcnow(), mock.Mock()) span.add_message_event(message_event) diff --git a/tests/unit/trace/test_exceptions_status.py b/tests/unit/trace/test_exceptions_status.py new file mode 100644 index 000000000..2d17349db --- /dev/null +++ b/tests/unit/trace/test_exceptions_status.py @@ -0,0 +1,50 @@ +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from google.rpc import code_pb2 + +from opencensus.trace import exceptions_status + + +class TestUtils(unittest.TestCase): + def test_cancelled(self): + self.assertEqual( + exceptions_status.CANCELLED.canonical_code, + code_pb2.CANCELLED + ) + + def test_invalid_url(self): + self.assertEqual( + exceptions_status.INVALID_URL.canonical_code, + code_pb2.INVALID_ARGUMENT + ) + + def test_timeout(self): + self.assertEqual( + exceptions_status.TIMEOUT.canonical_code, + code_pb2.DEADLINE_EXCEEDED + ) + + def test_unknown(self): + status = exceptions_status.unknown(Exception) + self.assertEqual( + status.canonical_code, + code_pb2.UNKNOWN + ) + self.assertEqual( + status.description, + str(Exception) + ) diff --git a/tests/unit/trace/test_execution_context.py b/tests/unit/trace/test_execution_context.py index 5f884b9fc..9df5e7d1f 100644 --- a/tests/unit/trace/test_execution_context.py +++ b/tests/unit/trace/test_execution_context.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import threading import unittest + import mock -import threading from opencensus.trace import execution_context diff --git a/tests/unit/trace/test_ext_utils.py b/tests/unit/trace/test_ext_utils.py index 96092fcb5..e322158a7 100644 --- a/tests/unit/trace/test_ext_utils.py +++ b/tests/unit/trace/test_ext_utils.py @@ -15,6 +15,7 @@ import unittest import mock +from google.rpc import code_pb2 from opencensus.trace import utils @@ -52,9 +53,9 @@ def test_disable_tracing_url_default(self): def test_disable_tracing_url_explicit(self): url = 'http://127.0.0.1:8080/test_no_tracing' - blacklist_paths = ['test_no_tracing'] + excludelist_paths = ['test_no_tracing'] - disable_tracing = utils.disable_tracing_url(url, blacklist_paths) + disable_tracing = utils.disable_tracing_url(url, excludelist_paths) self.assertTrue(disable_tracing) def test_disable_tracing_hostname_default(self): @@ -64,12 +65,78 @@ def test_disable_tracing_hostname_default(self): self.assertFalse(disable_tracing) def test_disable_tracing_hostname_explicit(self): - blacklist_paths = ['127.0.0.1', '192.168.0.1:80'] + excludelist_paths = ['127.0.0.1', '192.168.0.1:80'] url = '127.0.0.1:8080' - disable_tracing = utils.disable_tracing_hostname(url, blacklist_paths) + disable_tracing = utils.disable_tracing_hostname( + url, excludelist_paths) self.assertFalse(disable_tracing) url = '127.0.0.1:80' - disable_tracing = utils.disable_tracing_hostname(url, blacklist_paths) + disable_tracing = utils.disable_tracing_hostname( + url, excludelist_paths) self.assertFalse(disable_tracing) + + def test_grpc_code_from_http_code(self): + test_cases = [ + { + 'http_code': 0, + 'grpc_code': code_pb2.UNKNOWN, + }, + { + 'http_code': 200, + 'grpc_code': code_pb2.OK, + }, + { + 'http_code': 399, + 'grpc_code': code_pb2.OK, + }, + { + 'http_code': 400, + 'grpc_code': code_pb2.INVALID_ARGUMENT, + }, + { + 'http_code': 504, + 'grpc_code': code_pb2.DEADLINE_EXCEEDED, + }, + { + 'http_code': 404, + 'grpc_code': code_pb2.NOT_FOUND, + }, + { + 'http_code': 403, + 'grpc_code': code_pb2.PERMISSION_DENIED, + }, + { + 'http_code': 401, + 'grpc_code': code_pb2.UNAUTHENTICATED, + }, + { + 'http_code': 429, + 'grpc_code': code_pb2.RESOURCE_EXHAUSTED, + }, + { + 'http_code': 501, + 'grpc_code': code_pb2.UNIMPLEMENTED, + }, + { + 'http_code': 503, + 'grpc_code': code_pb2.UNAVAILABLE, + }, + { + 'http_code': 600, + 'grpc_code': code_pb2.UNKNOWN, + }, + ] + + for test_case in test_cases: + status = utils.status_from_http_code(test_case['http_code']) + self.assertEqual( + status.canonical_code, + test_case['grpc_code'], + 'HTTP: {} / GRPC: expected = {}, actual = {}'.format( + test_case['http_code'], + test_case['grpc_code'], + status.canonical_code, + ) + ) diff --git a/tests/unit/trace/test_span.py b/tests/unit/trace/test_span.py index 2d1c6ecd3..75637ad88 100644 --- a/tests/unit/trace/test_span.py +++ b/tests/unit/trace/test_span.py @@ -12,21 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import OrderedDict import datetime import unittest +from collections import OrderedDict import mock - from google.rpc import code_pb2 from opencensus.common import utils -from opencensus.trace.span import BoundedDict -from opencensus.trace.span import BoundedList +from opencensus.trace.span import BoundedDict, BoundedList from opencensus.trace.stack_trace import StackTrace from opencensus.trace.status import Status -from opencensus.trace.time_event import Annotation -from opencensus.trace.time_event import MessageEvent +from opencensus.trace.time_event import Annotation, MessageEvent class TestSpan(unittest.TestCase): @@ -45,6 +42,7 @@ def _make_one(self, *args, **kw): def test_constructor_defaults(self): span_id = 'test_span_id' span_name = 'test_span_name' + status = Status.as_ok() patch = mock.patch( 'opencensus.trace.span.generate_span_id', return_value=span_id) @@ -56,6 +54,7 @@ def test_constructor_defaults(self): self.assertEqual(span.span_id, span_id) self.assertIsNone(span.parent_span) self.assertEqual(span.attributes, {}) + self.assertDictEqual(span.status.__dict__, status.__dict__) self.assertIsNone(span.start_time) self.assertIsNone(span.end_time) self.assertEqual(span.children, []) @@ -181,6 +180,24 @@ def test_add_link(self): self.assertEqual(len(span.links), 1) + def test_set_status(self): + span_name = 'test_span_name' + span = self._make_one(span_name) + status = mock.Mock() + + with self.assertRaises(TypeError): + span.set_status(status) + + code = 1 + message = 'ok' + details = {'object': 'ok'} + status = Status(code=code, message=message, details=details) + span.set_status(status) + + self.assertEqual(span.status.canonical_code, code) + self.assertEqual(span.status.description, message) + self.assertEqual(span.status.details, details) + def test_start(self): span_name = 'root_span' span = self._make_one(span_name) @@ -278,8 +295,8 @@ def test_exception_in_span(self): self.assertIsNotNone(stack_frame['load_module']['build_id']['value']) self.assertIsNotNone(root_span.status) - self.assertEqual(root_span.status.message, exception_message) - self.assertEqual(root_span.status.code, code_pb2.UNKNOWN) + self.assertEqual(root_span.status.description, exception_message) + self.assertEqual(root_span.status.canonical_code, code_pb2.UNKNOWN) class Test_format_span_json(unittest.TestCase): diff --git a/tests/unit/trace/test_span_context.py b/tests/unit/trace/test_span_context.py index 13602e3f0..d3d1a279b 100644 --- a/tests/unit/trace/test_span_context.py +++ b/tests/unit/trace/test_span_context.py @@ -13,6 +13,7 @@ # limitations under the License. import unittest + from opencensus.trace import span_context as span_context_module from opencensus.trace.trace_options import TraceOptions from opencensus.trace.tracestate import Tracestate diff --git a/tests/unit/trace/test_span_data.py b/tests/unit/trace/test_span_data.py index eb8109ef6..60b23ba9b 100644 --- a/tests/unit/trace/test_span_data.py +++ b/tests/unit/trace/test_span_data.py @@ -16,12 +16,9 @@ import unittest from opencensus.common import utils -from opencensus.trace import link -from opencensus.trace import span_context +from opencensus.trace import link, span_context from opencensus.trace import span_data as span_data_module -from opencensus.trace import stack_trace -from opencensus.trace import status -from opencensus.trace import time_event +from opencensus.trace import stack_trace, status, time_event class TestSpanData(unittest.TestCase): diff --git a/tests/unit/trace/test_status.py b/tests/unit/trace/test_status.py index 1c4a8e816..c94ad903a 100644 --- a/tests/unit/trace/test_status.py +++ b/tests/unit/trace/test_status.py @@ -25,10 +25,21 @@ def test_constructor(self): message = 'test message' status = status_module.Status(code=code, message=message) - self.assertEqual(status.code, code) - self.assertEqual(status.message, message) + self.assertEqual(status.canonical_code, code) + self.assertEqual(status.description, message) self.assertIsNone(status.details) + def test_format_status_json_without_message(self): + code = 100 + status = status_module.Status(code=code) + status_json = status.format_status_json() + + expected_status_json = { + 'code': code + } + + self.assertEqual(expected_status_json, status_json) + def test_format_status_json_with_details(self): code = 100 message = 'test message' @@ -64,9 +75,23 @@ def test_format_status_json_without_details(self): self.assertEqual(expected_status_json, status_json) + def test_is_ok(self): + status = status_module.Status.as_ok() + self.assertTrue(status.is_ok) + + status = status_module.Status(code=code_pb2.UNKNOWN) + self.assertFalse(status.is_ok) + def test_create_from_exception(self): message = 'test message' exc = ValueError(message) status = status_module.Status.from_exception(exc) - self.assertEqual(status.message, message) - self.assertEqual(status.code, code_pb2.UNKNOWN) + self.assertEqual(status.description, message) + self.assertEqual(status.canonical_code, code_pb2.UNKNOWN) + self.assertIsNone(status.details) + + def test_create_as_ok(self): + status = status_module.Status.as_ok() + self.assertEqual(status.canonical_code, code_pb2.OK) + self.assertIsNone(status.description) + self.assertIsNone(status.details) diff --git a/tests/unit/trace/test_time_event.py b/tests/unit/trace/test_time_event.py index a1e753181..063dfb4ad 100644 --- a/tests/unit/trace/test_time_event.py +++ b/tests/unit/trace/test_time_event.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime import unittest +from datetime import datetime import mock diff --git a/tests/unit/trace/test_tracer.py b/tests/unit/trace/test_tracer.py index a4dbc9f0d..82b9a16a3 100644 --- a/tests/unit/trace/test_tracer.py +++ b/tests/unit/trace/test_tracer.py @@ -16,8 +16,7 @@ import mock -from opencensus.trace import samplers -from opencensus.trace import span_data +from opencensus.trace import samplers, span_data from opencensus.trace import tracer as tracer_module diff --git a/tests/unit/trace/test_tracestate.py b/tests/unit/trace/test_tracestate.py index 2aeac8f64..170f248ef 100644 --- a/tests/unit/trace/test_tracestate.py +++ b/tests/unit/trace/test_tracestate.py @@ -14,9 +14,10 @@ import unittest +from opencensus.trace.propagation.tracestate_string_format import ( + TracestateStringFormatter, +) from opencensus.trace.tracestate import Tracestate -from opencensus.trace.propagation.tracestate_string_format \ - import TracestateStringFormatter formatter = TracestateStringFormatter() diff --git a/tests/unit/trace/tracers/test_context_tracer.py b/tests/unit/trace/tracers/test_context_tracer.py index f05d048ae..1898fb37f 100644 --- a/tests/unit/trace/tracers/test_context_tracer.py +++ b/tests/unit/trace/tracers/test_context_tracer.py @@ -16,9 +16,8 @@ import mock +from opencensus.trace import execution_context, span from opencensus.trace.tracers import context_tracer -from opencensus.trace import span -from opencensus.trace import execution_context class TestContextTracer(unittest.TestCase): diff --git a/tox.ini b/tox.ini index 2901a097a..9802276e5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,55 +1,74 @@ [tox] -envlist = py{27,34,35,36,37}-unit, py37-lint, py37-setup, py{27,34,35,36,37}-cover, py37-docs +envlist = + py{27,35,36,37,38,39}-unit + py39-bandit + py39-lint + py39-setup + py39-docs + +[constants] +unit-base-command = py.test --quiet --cov={envdir}/opencensus --cov=context --cov=contrib --cov-report term-missing --cov-config=.coveragerc --cov-fail-under=90 --ignore=contrib/opencensus-ext-datadog tests/unit/ context/ contrib/ [testenv] install_command = python -m pip install {opts} {packages} -deps = - py{27,34,35,36,37}-unit,py37-lint: mock - py{27,34,35,36,37}-unit,py37-lint: pytest - py{27,34,35,36,37}-unit,py37-lint: pytest-cov - py{27,34,35,36,37}-unit,py37-lint: retrying - py{27,34,35,36,37}-unit,py37-lint: unittest2 - py{27,34,35,36,37}-unit,py37-lint,py37-setup,py{27,34,35,36,37}-cover,py37-docs: -e context/opencensus-context - py{27,34,35,36,37}-unit,py37-lint,py37-docs: -e contrib/opencensus-correlation - py{27,34,35,36,37}-unit,py37-lint,py37-docs: -e . - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-azure - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-dbapi - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-django - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-flask - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-gevent - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-grpc - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-httplib - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-jaeger - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-logging - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-mysql - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-ocagent - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-postgresql - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-prometheus - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-pymongo - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-pymysql - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-pyramid - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-requests - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-sqlalchemy - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-stackdriver - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-threading - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-zipkin - py{27,34,35,36,37}-unit,py37-lint: -e contrib/opencensus-ext-google-cloud-clientlibs - py37-lint: flake8 - py37-setup: docutils - py37-setup: pygments - py{27,34,35,36,37}-cover: coverage - py{27,34,35,36,37}-cover: pytest-cov - py37-docs: setuptools >= 36.4.0 - py37-docs: sphinx >= 1.6.3 +deps = + unit,lint: mock==3.0.5 + unit,lint: pytest==4.6.4 + unit,lint: pytest-cov + unit,lint: retrying + unit,lint: unittest2 + py27-unit: markupsafe==1.1.1 + py35-unit: markupsafe==1.1.1 + py3{6,7,8,9}-unit: markupsafe==2.0.1 # https://github.com/pallets/markupsafe/issues/282 + bandit: bandit + unit,lint,setup,docs,bandit: -e context/opencensus-context + unit,lint,docs,bandit: -e contrib/opencensus-correlation + unit,lint,docs,bandit: protobuf==3.17.3 # https://github.com/protocolbuffers/protobuf/issues/8984 + unit,lint,docs,bandit: -e . + unit,lint,bandit: -e contrib/opencensus-ext-azure + ; unit,lint: -e contrib/opencensus-ext-datadog + unit,lint,bandit: -e contrib/opencensus-ext-dbapi + unit,lint,bandit: -e contrib/opencensus-ext-django + py3{6,7,8,9}-unit,lint,bandit: -e contrib/opencensus-ext-fastapi + unit,lint,bandit: -e contrib/opencensus-ext-flask + unit,lint,bandit: -e contrib/opencensus-ext-gevent + unit,lint,bandit: -e contrib/opencensus-ext-grpc + unit,lint,bandit: -e contrib/opencensus-ext-httplib + unit,lint,bandit: -e contrib/opencensus-ext-jaeger + unit,lint,bandit: -e contrib/opencensus-ext-logging + unit,lint,bandit: -e contrib/opencensus-ext-mysql + unit,lint,bandit: -e contrib/opencensus-ext-ocagent + unit,lint,bandit: -e contrib/opencensus-ext-postgresql + py{36,37,38,39}-unit,lint,bandit: prometheus-client==0.13.1 + unit,lint,bandit: -e contrib/opencensus-ext-prometheus + unit,lint,bandit: -e contrib/opencensus-ext-pymongo + unit,lint,bandit: -e contrib/opencensus-ext-pymysql + unit,lint,bandit: -e contrib/opencensus-ext-pyramid + unit,lint,bandit: -e contrib/opencensus-ext-requests + py3{7,8,9}-unit,lint,bandit: -e contrib/opencensus-ext-httpx + unit,lint,bandit: -e contrib/opencensus-ext-sqlalchemy + py3{6,7,8,9}-unit,lint,bandit: -e contrib/opencensus-ext-stackdriver + unit,lint,bandit: -e contrib/opencensus-ext-threading + unit,lint,bandit: -e contrib/opencensus-ext-zipkin + unit,lint,bandit: -e contrib/opencensus-ext-google-cloud-clientlibs + lint: flake8 ~= 4.0.1 + lint: isort ~= 4.3.21 + setup: docutils + setup: pygments + docs: setuptools >= 36.4.0 + docs: sphinx >= 1.6.3 -commands = - py{27,34,35,36,37}-unit: py.test --quiet --cov={envdir}/opencensus --cov=context --cov=contrib --cov-report= --cov-config=.coveragerc --cov-fail-under=97 tests/unit/ context/ contrib/ - ; TODO: System tests - py37-lint: flake8 context/ contrib/ opencensus/ tests/ examples/ - py37-setup: python setup.py check --restructuredtext --strict - py{27,34,35,36,37}-cover: coverage report --show-missing --fail-under=97 - py{27,34,35,36,37}-cover: coverage erase - py37-docs: bash ./scripts/update_docs.sh - ; TODO deployment +commands = + py{27,34,35}-unit: {[constants]unit-base-command} --ignore=contrib/opencensus-ext-stackdriver --ignore=contrib/opencensus-ext-flask --ignore=contrib/opencensus-ext-httpx --ignore=contrib/opencensus-ext-fastapi + py36-unit: {[constants]unit-base-command} --ignore=contrib/opencensus-ext-httpx + py3{7,8,9}-unit: {[constants]unit-base-command} + ; TODO system tests + lint: isort --check-only --diff --recursive . + lint: flake8 context/ contrib/ opencensus/ tests/ examples/ + ; lint: - bash ./scripts/pylint.sh + bandit: bandit -r context/ contrib/ opencensus/ -lll -q + py39-setup: python setup.py check --restructuredtext --strict + py39-docs: bash ./scripts/update_docs.sh + ; TODO deployment \ No newline at end of file