From ee0f3be8b371a2d2ca9aa40929bbbed330819c1e Mon Sep 17 00:00:00 2001 From: Som <111349830+someshwaranM@users.noreply.github.com> Date: Fri, 19 Jan 2024 04:22:05 +0530 Subject: [PATCH 001/103] added a note on centralized agent config support for AWS Lambda (#1954) --- docs/serverless-lambda.asciidoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/serverless-lambda.asciidoc b/docs/serverless-lambda.asciidoc index 48c091390..732abb2b4 100644 --- a/docs/serverless-lambda.asciidoc +++ b/docs/serverless-lambda.asciidoc @@ -5,6 +5,11 @@ The Python APM Agent can be used with AWS Lambda to monitor the execution of your AWS Lambda functions. +``` +Note: The Centralized Agent Configuration on the Elasticsearch APM currently does NOT support AWS Lambda. +``` + + [float] ==== Prerequisites From e6039552cbc6805ff29980433671173aa754585d Mon Sep 17 00:00:00 2001 From: Gabriel Figueiredo <53541827+gabriel-f-santos@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:32:18 -0300 Subject: [PATCH 002/103] fix: setup middleware without client argument for starlette (#1952) --- .ci/.matrix_exclude.yml | 2 ++ elasticapm/contrib/starlette/__init__.py | 2 +- tests/contrib/asyncio/starlette_tests.py | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.ci/.matrix_exclude.yml b/.ci/.matrix_exclude.yml index eaf0fa82b..d0c69e15c 100644 --- a/.ci/.matrix_exclude.yml +++ b/.ci/.matrix_exclude.yml @@ -279,3 +279,5 @@ exclude: FRAMEWORK: pyodbc-newest # error on wheel - VERSION: python-3.12 FRAMEWORK: cassandra-newest # c extension issue + - VERSION: python-3.12 + FRAMEWORK: starlette-newest # waiting for 3.12.2 for this fix: https://github.com/python/cpython/pull/111221 diff --git a/elasticapm/contrib/starlette/__init__.py b/elasticapm/contrib/starlette/__init__.py index a6262ba86..fcc5dc3b4 100644 --- a/elasticapm/contrib/starlette/__init__.py +++ b/elasticapm/contrib/starlette/__init__.py @@ -105,7 +105,7 @@ class ElasticAPM: >>> elasticapm.capture_message('hello, world!') """ - def __init__(self, app: ASGIApp, client: Optional[Client], **kwargs) -> None: + def __init__(self, app: ASGIApp, client: Optional[Client] = None, **kwargs) -> None: """ Args: diff --git a/tests/contrib/asyncio/starlette_tests.py b/tests/contrib/asyncio/starlette_tests.py index 5f4c070bd..fcd7d0dee 100644 --- a/tests/contrib/asyncio/starlette_tests.py +++ b/tests/contrib/asyncio/starlette_tests.py @@ -534,3 +534,11 @@ def test_transaction_active_in_base_exception_handler(app, elasticapm_client): assert exc.transaction_id assert len(elasticapm_client.events[constants.TRANSACTION]) == 1 + + +def test_middleware_without_client_arg(): + with mock.patch.dict("os.environ", {"ELASTIC_APM_SERVICE_NAME": "foo"}): + app = Starlette() + elasticapm = ElasticAPM(app) + + assert elasticapm.client.config.service_name == "foo" From a0c4c806a44ecf8316c8c28c1fe44c6772386c8f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 26 Jan 2024 09:38:24 +0100 Subject: [PATCH 003/103] updatecli: dynamic specs (#1957) --- .ci/updatecli.d/update-gherkin-specs.yml | 117 ----------------- .ci/updatecli.d/update-json-specs.yml | 122 ------------------ .ci/updatecli.d/update-specs.yml | 104 --------------- .../updatecli.d/update-gherkin-specs.yml | 84 ++++++++++++ .../updatecli.d/update-json-specs.yml | 84 ++++++++++++ .ci/updatecli/updatecli.d/update-specs.yml | 86 ++++++++++++ .ci/updatecli/values.yml | 14 ++ .github/workflows/updatecli.yml | 3 +- 8 files changed, 270 insertions(+), 344 deletions(-) delete mode 100644 .ci/updatecli.d/update-gherkin-specs.yml delete mode 100644 .ci/updatecli.d/update-json-specs.yml delete mode 100644 .ci/updatecli.d/update-specs.yml create mode 100644 .ci/updatecli/updatecli.d/update-gherkin-specs.yml create mode 100644 .ci/updatecli/updatecli.d/update-json-specs.yml create mode 100644 .ci/updatecli/updatecli.d/update-specs.yml create mode 100644 .ci/updatecli/values.yml diff --git a/.ci/updatecli.d/update-gherkin-specs.yml b/.ci/updatecli.d/update-gherkin-specs.yml deleted file mode 100644 index 8deb269fc..000000000 --- a/.ci/updatecli.d/update-gherkin-specs.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: update-gherkin-specs -pipelineid: update-gherkin-specs -title: synchronize gherkin specs - -scms: - default: - kind: github - spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' - owner: elastic - repository: apm-agent-python - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' - branch: main - -sources: - sha: - kind: file - spec: - file: 'https://github.com/elastic/apm/commit/main.patch' - matchpattern: "^From\\s([0-9a-f]{40})\\s" - transformers: - - findsubmatch: - pattern: "[0-9a-f]{40}" - - api_key.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/api_key.feature - azure_app_service_metadata.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/azure_app_service_metadata.feature - azure_functions_metadata.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/azure_functions_metadata.feature - otel_bridge.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/otel_bridge.feature - outcome.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/outcome.feature - user_agent.feature: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/gherkin-specs/user_agent.feature - -actions: - pr: - kind: "github/pullrequest" - scmid: default - title: '[Automation] Update Gherkin specs' - spec: - automerge: false - draft: false - labels: - - "automation" - description: |- - ### What - APM agent Gherkin specs automatic sync - ### Why - *Changeset* - * https://github.com/elastic/apm/commit/{{ source "sha" }} - -targets: - api_key.feature: - name: api_key.feature - scmid: default - sourceid: api_key.feature - kind: file - spec: - file: tests/bdd/features/api_key.feature - forcecreate: true - azure_app_service_metadata.feature: - name: azure_app_service_metadata.feature - scmid: default - sourceid: azure_app_service_metadata.feature - kind: file - spec: - file: tests/bdd/features/azure_app_service_metadata.feature - forcecreate: true - azure_functions_metadata.feature: - name: azure_functions_metadata.feature - scmid: default - sourceid: azure_functions_metadata.feature - kind: file - spec: - file: tests/bdd/features/azure_functions_metadata.feature - forcecreate: true - otel_bridge.feature: - name: otel_bridge.feature - scmid: default - sourceid: otel_bridge.feature - kind: file - spec: - file: tests/bdd/features/otel_bridge.feature - forcecreate: true - outcome.feature: - name: outcome.feature - scmid: default - sourceid: outcome.feature - kind: file - spec: - file: tests/bdd/features/outcome.feature - forcecreate: true - user_agent.feature: - name: user_agent.feature - scmid: default - sourceid: user_agent.feature - kind: file - spec: - file: tests/bdd/features/user_agent.feature - forcecreate: true diff --git a/.ci/updatecli.d/update-json-specs.yml b/.ci/updatecli.d/update-json-specs.yml deleted file mode 100644 index 13d25c834..000000000 --- a/.ci/updatecli.d/update-json-specs.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: update-json-specs -pipelineid: update-json-specs -title: synchronize json specs - -scms: - default: - kind: github - spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' - owner: elastic - repository: apm-agent-python - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' - branch: main - -sources: - sha: - kind: file - spec: - file: 'https://github.com/elastic/apm/commit/main.patch' - matchpattern: "^From\\s([0-9a-f]{40})\\s" - transformers: - - findsubmatch: - pattern: "[0-9a-f]{40}" - - container_metadata_discovery.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/container_metadata_discovery.json - service_resource_inference.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/service_resource_inference.json - span_types.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/span_types.json - sql_signature_examples.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/sql_signature_examples.json - sql_token_examples.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/sql_token_examples.json - w3c_distributed_tracing.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/w3c_distributed_tracing.json - wildcard_matcher_tests.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/wildcard_matcher_tests.json - -actions: - pr: - kind: "github/pullrequest" - scmid: default - title: '[Automation] Update JSON specs' - spec: - automerge: false - draft: false - labels: - - "automation" - description: |- - ### What - APM agent specs automatic sync - ### Why - *Changeset* - * https://github.com/elastic/apm/commit/{{ source "sha" }} - -targets: - container_metadata_discovery.json: - name: container_metadata_discovery.json - scmid: default - sourceid: container_metadata_discovery.json - kind: file - spec: - file: tests/upstream/json-specs/container_metadata_discovery.json - service_resource_inference.json: - name: service_resource_inference.json - scmid: default - sourceid: service_resource_inference.json - kind: file - spec: - file: tests/upstream/json-specs/service_resource_inference.json - span_types.json: - name: span_types.json - scmid: default - sourceid: span_types.json - kind: file - spec: - file: tests/upstream/json-specs/span_types.json - sql_signature_examples.json: - name: sql_signature_examples.json - scmid: default - sourceid: sql_signature_examples.json - kind: file - spec: - file: tests/upstream/json-specs/sql_signature_examples.json - sql_token_examples.json: - name: sql_token_examples.json - scmid: default - sourceid: sql_token_examples.json - kind: file - spec: - file: tests/upstream/json-specs/sql_token_examples.json - w3c_distributed_tracing.json: - name: w3c_distributed_tracing.json - scmid: default - sourceid: w3c_distributed_tracing.json - kind: file - spec: - file: tests/upstream/json-specs/w3c_distributed_tracing.json - wildcard_matcher_tests.json: - name: wildcard_matcher_tests.json - scmid: default - sourceid: wildcard_matcher_tests.json - kind: file - spec: - file: tests/upstream/json-specs/wildcard_matcher_tests.json diff --git a/.ci/updatecli.d/update-specs.yml b/.ci/updatecli.d/update-specs.yml deleted file mode 100644 index ab3bd34c7..000000000 --- a/.ci/updatecli.d/update-specs.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: update-specs -pipelineid: update-schema-specs -title: synchronize schema specs - -scms: - default: - kind: github - spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' - owner: elastic - repository: apm-agent-python - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' - branch: main - -sources: - sha: - kind: file - spec: - file: 'https://github.com/elastic/apm-data/commit/main.patch' - matchpattern: "^From\\s([0-9a-f]{40})\\s" - transformers: - - findsubmatch: - pattern: "[0-9a-f]{40}" - error.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/error.json - metadata.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/metadata.json - metricset.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/metricset.json - span.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/span.json - transaction.json: - kind: file - spec: - file: https://raw.githubusercontent.com/elastic/apm-data/main/input/elasticapm/docs/spec/v2/transaction.json - -actions: - pr: - kind: "github/pullrequest" - scmid: default - title: '[Automation] Update JSON schema specs' - spec: - automerge: false - draft: false - labels: - - "automation" - description: |- - ### What - APM agent json schema automatic sync - ### Why - *Changeset* - * https://github.com/elastic/apm-data/commit/{{ source "sha" }} - -targets: - error.json: - name: error.json - scmid: default - sourceid: error.json - kind: file - spec: - file: tests/upstream/json-specs/error.json - forcecreate: true - metadata.json: - name: metadata.json - scmid: default - sourceid: metadata.json - kind: file - spec: - file: tests/upstream/json-specs/metadata.json - forcecreate: true - metricset.json: - name: metricset.json - scmid: default - sourceid: metricset.json - kind: file - spec: - file: tests/upstream/json-specs/metricset.json - forcecreate: true - span.json: - name: span.json - scmid: default - sourceid: span.json - kind: file - spec: - file: tests/upstream/json-specs/span.json - forcecreate: true - transaction.json: - name: transaction.json - scmid: default - sourceid: transaction.json - kind: file - spec: - file: tests/upstream/json-specs/transaction.json - forcecreate: true diff --git a/.ci/updatecli/updatecli.d/update-gherkin-specs.yml b/.ci/updatecli/updatecli.d/update-gherkin-specs.yml new file mode 100644 index 000000000..f12ece861 --- /dev/null +++ b/.ci/updatecli/updatecli.d/update-gherkin-specs.yml @@ -0,0 +1,84 @@ +name: update-gherkin-specs +pipelineid: update-gherkin-specs + +scms: + default: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: "{{ .github.branch }}" + apm: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: "{{ .github.owner }}" + repository: "{{ .github.apm_repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: "{{ .github.branch }}" + +sources: + sha: + kind: file + spec: + file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ .github.branch }}.patch' + matchpattern: "^From\\s([0-9a-f]{40})\\s" + transformers: + - findsubmatch: + pattern: "[0-9a-f]{40}" + pull_request: + kind: shell + dependson: + - sha + spec: + command: gh api /repos/{{ .github.owner }}/{{ .github.apm_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' + environments: + - name: GITHUB_TOKEN + - name: PATH + agents-gherkin-specs-tarball: + kind: shell + scmid: apm + dependson: + - sha + spec: + command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/gherkin-specs.tgz . + environments: + - name: PATH + workdir: "{{ .specs.apm_gherkin_path }}" + +actions: + pr: + kind: "github/pullrequest" + scmid: default + spec: + automerge: false + draft: false + labels: + - "automation" + description: |- + ### What + APM agent Gherkin specs automatic sync + + ### Why + *Changeset* + * {{ source "pull_request" }} + * https://github.com/elastic/apm/commit/{{ source "sha" }} + title: '[Automation] Update Gherkin specs' + +targets: + agent-gherkin-specs: + name: APM agent gherkin specs {{ source "sha" }} + scmid: default + disablesourceinput: true + kind: shell + spec: + # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. + # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target + command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/gherkin-specs.tgz && git --no-pager diff' + workdir: "{{ .apm_agent.gherkin_specs_path }}" diff --git a/.ci/updatecli/updatecli.d/update-json-specs.yml b/.ci/updatecli/updatecli.d/update-json-specs.yml new file mode 100644 index 000000000..e05aaecdb --- /dev/null +++ b/.ci/updatecli/updatecli.d/update-json-specs.yml @@ -0,0 +1,84 @@ +name: update-json-specs +pipelineid: update-json-specs + +scms: + default: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: "{{ .github.branch }}" + apm: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: "{{ .github.owner }}" + repository: "{{ .github.apm_repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: "{{ .github.branch }}" + +sources: + sha: + kind: file + spec: + file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ .github.branch }}.patch' + matchpattern: "^From\\s([0-9a-f]{40})\\s" + transformers: + - findsubmatch: + pattern: "[0-9a-f]{40}" + pull_request: + kind: shell + dependson: + - sha + spec: + command: gh api /repos/{{ .github.owner }}/{{ .github.apm_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' + environments: + - name: GITHUB_TOKEN + - name: PATH + agents-json-specs-tarball: + kind: shell + scmid: apm + dependson: + - sha + spec: + command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-specs.tgz . + environments: + - name: PATH + workdir: "{{ .specs.apm_json_path }}" + +actions: + pr: + kind: "github/pullrequest" + scmid: default + spec: + automerge: false + draft: false + labels: + - "automation" + description: |- + ### What + APM agent specs automatic sync + + ### Why + *Changeset* + * {{ source "pull_request" }} + * https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ source "sha" }} + title: '[Automation] Update JSON specs' + +targets: + agent-json-specs: + name: APM agent json specs {{ source "sha" }} + scmid: default + disablesourceinput: true + kind: shell + spec: + # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. + # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target + command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-specs.tgz && git --no-pager diff' + workdir: "{{ .apm_agent.json_specs_path }}" diff --git a/.ci/updatecli/updatecli.d/update-specs.yml b/.ci/updatecli/updatecli.d/update-specs.yml new file mode 100644 index 000000000..554140da2 --- /dev/null +++ b/.ci/updatecli/updatecli.d/update-specs.yml @@ -0,0 +1,86 @@ +name: update-specs +pipelineid: update-schema-specs + +scms: + default: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: "{{ .github.branch }}" + + apm-data: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: "{{ .github.owner }}" + repository: "{{ .github.apm_data_repository }}" + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: "{{ .github.branch }}" + +sources: + sha: + kind: file + spec: + file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_data_repository }}/commit/{{ .github.branch }}.patch' + matchpattern: "^From\\s([0-9a-f]{40})\\s" + transformers: + - findsubmatch: + pattern: "[0-9a-f]{40}" + pull_request: + kind: shell + dependson: + - sha + spec: + command: gh api /repos/{{ .github.owner }}/{{ .github.apm_data_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' + environments: + - name: GITHUB_TOKEN + - name: PATH + agent-specs-tarball: + kind: shell + scmid: apm-data + dependson: + - sha + spec: + command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-schema.tgz . + environments: + - name: PATH + workdir: "{{ .specs.apm_data_path }}" + +actions: + pr: + kind: "github/pullrequest" + scmid: default + sourceid: sha + spec: + automerge: false + draft: false + labels: + - "automation" + description: |- + ### What + APM agent json server schema automatic sync + + ### Why + *Changeset* + * {{ source "pull_request" }} + * https://github.com/{{ .github.owner }}/{{ .github.apm_data_repository }}/commit/{{ source "sha" }} + title: '[Automation] Update JSON server schema specs' + +targets: + agent-json-schema: + name: APM agent json server schema {{ source "sha" }} + scmid: default + disablesourceinput: true + kind: shell + spec: + # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. + # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target + command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-schema.tgz && git --no-pager diff' + workdir: "{{ .apm_agent.server_schema_specs_path }}" diff --git a/.ci/updatecli/values.yml b/.ci/updatecli/values.yml new file mode 100644 index 000000000..b0b58d73e --- /dev/null +++ b/.ci/updatecli/values.yml @@ -0,0 +1,14 @@ +github: + owner: "elastic" + repository: "apm-agent-python" + apm_repository: "apm" + apm_data_repository: "apm-data" + branch: "main" +specs: + apm_data_path: "input/elasticapm/docs/spec/v2" + apm_json_path: "tests/agents/json-specs" + apm_gherkin_path: "tests/agents/gherkin-specs" +apm_agent: + gherkin_specs_path: "tests/bdd/features" + json_specs_path: "tests/upstream/json-specs" + server_schema_specs_path: "tests/upstream/json-specs" \ No newline at end of file diff --git a/.github/workflows/updatecli.yml b/.github/workflows/updatecli.yml index 2101ec798..22598200b 100644 --- a/.github/workflows/updatecli.yml +++ b/.github/workflows/updatecli.yml @@ -18,7 +18,8 @@ jobs: vaultUrl: ${{ secrets.VAULT_ADDR }} vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - pipeline: .ci/updatecli.d + pipeline: .ci/updatecli/updatecli.d + values: .ci/updatecli/values.yml - if: failure() uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current with: From 7c4467fdbea162f6dc66fc075abfd3e71b1d63e1 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Mon, 29 Jan 2024 08:40:37 +0100 Subject: [PATCH 004/103] Add ability to run full matrix manually and on push (#1958) * Add ability to run full matrix manually and on releases * Fix typo * Run full matrix on push instead in the release * Re-add workflow_call trigger again --- .github/workflows/test.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 363dec4a6..0246189b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,12 @@ on: - "**/*.asciidoc" schedule: - cron: "0 2 * * *" + workflow_dispatch: + inputs: + full-matrix: + description: "Run the full matrix" + required: true + type: boolean jobs: build-distribution: @@ -31,9 +37,9 @@ jobs: uses: elastic/apm-pipeline-library/.github/actions/version-framework@current with: # Use .ci/.matrix_python_full.yml if it's a scheduled workflow, otherwise use .ci/.matrix_python.yml - versionsFile: .ci/.matrix_python${{ github.event_name == 'schedule' && '_full' || '' }}.yml + versionsFile: .ci/.matrix_python${{ (github.event_name == 'schedule' || github.event_name == 'push' || inputs.full-matrix) && '_full' || '' }}.yml # Use .ci/.matrix_framework_full.yml if it's a scheduled workflow, otherwise use .ci/.matrix_framework.yml - frameworksFile: .ci/.matrix_framework${{ github.event_name == 'schedule' && '_full' || '' }}.yml + frameworksFile: .ci/.matrix_framework${{ (github.event_name == 'schedule' || github.event_name == 'push' || inputs.full-matrix) && '_full' || '' }}.yml excludedFile: .ci/.matrix_exclude.yml - name: Split matrix shell: python From 5dba4f233db5053dbd3251bcc9eef47baec255eb Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Mon, 29 Jan 2024 21:31:03 +0100 Subject: [PATCH 005/103] Add notification if scheduled job or main fails (#1959) * Add notification if scheduled job fails * Update .github/workflows/test.yml --- .github/workflows/test.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0246189b0..7fab3fb26 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,7 +148,19 @@ jobs: - chunks-3 - windows steps: - - run: test $(echo '${{ toJSON(needs) }}' | jq -s 'map(.[].result) | all(.=="success")') = 'true' + - id: check + uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current + with: + needs: ${{ toJSON(needs) }} + - run: ${{ steps.check.outputs.isSuccess }} + - if: failure() && (github.event_name == 'schedule' || github.event_name == 'push') + uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current + with: + status: ${{ steps.check.outputs.status }} + vaultUrl: ${{ secrets.VAULT_ADDR }} + vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} + vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} + slackChannel: "#apm-agent-python" coverage: name: Combine & check coverage. From bda718ffc033220eb68469ef99e1aab2b0914eb3 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Mon, 29 Jan 2024 21:32:04 +0100 Subject: [PATCH 006/103] Add PR review comment command (#1960) * Add PR review comment command * Add to README.md * Update .github/workflows/README.md Co-authored-by: Victor Martinez * Update .github/workflows/test.yml --------- Co-authored-by: Victor Martinez --- .github/workflows/README.md | 1 + .github/workflows/matrix-command.yml | 49 ++++++++++++++++++++++++++++ .github/workflows/test.yml | 23 ++++++++++--- 3 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/matrix-command.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3cdfe70f0..c224d62b8 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -39,6 +39,7 @@ Once a PR has been opened then there are two different ways you can trigger buil 1. Commit based 1. UI based, any Elasticians can force a build through the GitHub UI +1. PR review comment-based, any Elastic employees can force a full matrix test run through a PR review comment with the following syntax: `/test matrix`. #### Branches diff --git a/.github/workflows/matrix-command.yml b/.github/workflows/matrix-command.yml new file mode 100644 index 000000000..f2c32658f --- /dev/null +++ b/.github/workflows/matrix-command.yml @@ -0,0 +1,49 @@ +name: matrix-command + +on: + pull_request_review: + types: + - submitted + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + +jobs: + command-validation: + if: startsWith(github.event.review.body, '/test matrix') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: write + steps: + - name: Is comment allowed? + uses: actions/github-script@v7 + with: + script: | + const actorPermission = (await github.rest.repos.getCollaboratorPermissionLevel({ + ...context.repo, + username: context.actor + })).data.permission + const isPermitted = ['write', 'admin'].includes(actorPermission) + if (!isPermitted) { + const errorMessage = 'Only users with write permission to the repository can run GitHub commands' + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: errorMessage, + }) + core.setFailed(errorMessage) + return + } + + test: + needs: + - command-validation + uses: ./.github/workflows/test.yml + with: + full-matrix: true + ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7fab3fb26..b57b8a023 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,16 @@ name: test # The name must be the same as in test-docs.yml on: - workflow_call: ~ + workflow_call: + inputs: + full-matrix: + description: "Run the full matrix" + required: true + type: boolean + ref: + description: "The git ref of elastic/apm-agent-python to run test workflow from." + required: false + type: string pull_request: paths-ignore: - "**/*.md" @@ -32,7 +41,9 @@ jobs: data: ${{ steps.split.outputs.data }} chunks: ${{ steps.split.outputs.chunks }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} - id: generate uses: elastic/apm-pipeline-library/.github/actions/version-framework@current with: @@ -113,7 +124,9 @@ jobs: FRAMEWORK: ${{ matrix.framework }} ASYNCIO: ${{ matrix.asyncio }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} - uses: actions/setup-python@v4 with: python-version: ${{ matrix.version }} @@ -168,7 +181,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} - uses: actions/setup-python@v4 with: From 98da7be161840576545dca45ca4c09d3a96e91e2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 30 Jan 2024 14:20:51 +0100 Subject: [PATCH 007/103] github-action: use wildcards for discovering all the workflows (#1961) --- .github/workflows/opentelemetry.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/opentelemetry.yml b/.github/workflows/opentelemetry.yml index ea858e655..84a6209ff 100644 --- a/.github/workflows/opentelemetry.yml +++ b/.github/workflows/opentelemetry.yml @@ -1,18 +1,16 @@ --- +# Look up results at https://ela.st/oblt-ci-cd-stats. +# There will be one service per GitHub repository, including the org name, and one Transaction per Workflow. name: OpenTelemetry Export Trace on: workflow_run: - workflows: - - pre-commit - - test - - test-reporter - - snapshoty - - release - - packages - - updatecli + workflows: [ "*" ] types: [completed] +permissions: + contents: read + jobs: otel-export-trace: runs-on: ubuntu-latest From c1c4528d52e7dca9e77e178e0987046b579a3ec9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 10:03:45 -0700 Subject: [PATCH 008/103] Bump certifi from 2023.11.17 to 2024.2.2 in /dev-utils (#1963) Bumps [certifi](https://github.com/certifi/python-certifi) from 2023.11.17 to 2024.2.2. - [Commits](https://github.com/certifi/python-certifi/compare/2023.11.17...2024.02.02) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev-utils/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-utils/requirements.txt b/dev-utils/requirements.txt index 21258b2ad..59008afc2 100644 --- a/dev-utils/requirements.txt +++ b/dev-utils/requirements.txt @@ -1,4 +1,4 @@ # These are the pinned requirements for the lambda layer/docker image -certifi==2023.11.17 +certifi==2024.2.2 urllib3==1.26.18 wrapt==1.14.1 From 889b11fc13b719b671102ee12d4b4d80ff07f827 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 5 Feb 2024 11:33:38 -0700 Subject: [PATCH 009/103] Fixup test suite (#1964) --- .ci/.matrix_exclude.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.ci/.matrix_exclude.yml b/.ci/.matrix_exclude.yml index d0c69e15c..d7d05f2af 100644 --- a/.ci/.matrix_exclude.yml +++ b/.ci/.matrix_exclude.yml @@ -184,8 +184,12 @@ exclude: # asyncpg - VERSION: pypy-3 FRAMEWORK: asyncpg-newest + - VERSION: pypy-3 + FRAMEWORK: asyncpg-0.28 - VERSION: python-3.6 FRAMEWORK: asyncpg-newest + - VERSION: python-3.6 + FRAMEWORK: asyncpg-0.28 # sanic - VERSION: pypy-3 FRAMEWORK: sanic-newest @@ -281,3 +285,7 @@ exclude: FRAMEWORK: cassandra-newest # c extension issue - VERSION: python-3.12 FRAMEWORK: starlette-newest # waiting for 3.12.2 for this fix: https://github.com/python/cpython/pull/111221 + - VERSION: python-3.12 + FRAMEWORK: starlette-0.14 # waiting for 3.12.2 for this fix: https://github.com/python/cpython/pull/111221 + - VERSION: python-3.12 + FRAMEWORK: starlette-0.13 # waiting for 3.12.2 for this fix: https://github.com/python/cpython/pull/111221 From 3b1a34b4406fe53932ab29439714848d35f4ccaa Mon Sep 17 00:00:00 2001 From: psydok <47638600+psydok@users.noreply.github.com> Date: Tue, 13 Feb 2024 06:00:12 +0500 Subject: [PATCH 010/103] Fix blocking of gRPC stream-to-stream requests when elasticapm is enabled (#1967) * Fix gprc support with streaming requests (#1966) * Update version.py --- .../contrib/grpc/async_server_interceptor.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/elasticapm/contrib/grpc/async_server_interceptor.py b/elasticapm/contrib/grpc/async_server_interceptor.py index 5af0c1372..e7c9b659f 100644 --- a/elasticapm/contrib/grpc/async_server_interceptor.py +++ b/elasticapm/contrib/grpc/async_server_interceptor.py @@ -33,20 +33,18 @@ import grpc import elasticapm -from elasticapm.contrib.grpc.server_interceptor import _ServicerContextWrapper, _wrap_rpc_behavior, get_trace_parent +from elasticapm.contrib.grpc.server_interceptor import _ServicerContextWrapper, get_trace_parent class _AsyncServerInterceptor(grpc.aio.ServerInterceptor): async def intercept_service(self, continuation, handler_call_details): - def transaction_wrapper(behavior, request_streaming, response_streaming): - async def _interceptor(request_or_iterator, context): - if request_streaming or response_streaming: # only unary-unary is supported - return behavior(request_or_iterator, context) + def wrap_unary_unary(behavior): + async def _interceptor(request, context): tp = get_trace_parent(handler_call_details) client = elasticapm.get_client() transaction = client.begin_transaction("request", trace_parent=tp) try: - result = behavior(request_or_iterator, _ServicerContextWrapper(context, transaction)) + result = behavior(request, _ServicerContextWrapper(context, transaction)) # This is so we can support both sync and async rpc functions if inspect.isawaitable(result): @@ -65,4 +63,12 @@ async def _interceptor(request_or_iterator, context): return _interceptor - return _wrap_rpc_behavior(await continuation(handler_call_details), transaction_wrapper) + handler = await continuation(handler_call_details) + if handler.request_streaming or handler.response_streaming: + return handler + + return grpc.unary_unary_rpc_method_handler( + wrap_unary_unary(handler.unary_unary), + request_deserializer=handler.request_deserializer, + response_serializer=handler.response_serializer, + ) From 3d7616c7c8a8665792c53ec0d09ad5b095225f30 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 15 Feb 2024 12:14:29 +0100 Subject: [PATCH 011/103] chore: enable dependabot version updates of github-actions yaml files (#1930) --- .github/dependabot.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index eb8155b22..eb1cff95b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,8 +8,24 @@ updates: # Check for updates once a week schedule: interval: "weekly" + day: "sunday" + time: "22:00" reviewers: - "elastic/apm-agent-python" ignore: - dependency-name: "urllib3" # ignore until lambda runtimes use OpenSSL 1.1.1+ versions: [">=2.0.0"] + + # GitHub actions + - package-ecosystem: "github-actions" + directory: "/" + reviewers: + - "elastic/observablt-ci" + schedule: + interval: "weekly" + day: "sunday" + time: "22:00" + groups: + github-actions: + patterns: + - "*" From da4c8d54d728d05088f11e6dd95d6557ca750dfb Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 16 Feb 2024 12:11:26 +0100 Subject: [PATCH 012/103] contrib/starlette: take into account body reading time (#1970) Create the transaction always before the body is read to avoid discrepancies between cases where the capture_body config on or off. Fix #1948 --- elasticapm/contrib/starlette/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/elasticapm/contrib/starlette/__init__.py b/elasticapm/contrib/starlette/__init__.py index fcc5dc3b4..ad26d7a0a 100644 --- a/elasticapm/contrib/starlette/__init__.py +++ b/elasticapm/contrib/starlette/__init__.py @@ -36,6 +36,7 @@ from typing import Dict, Optional import starlette +from starlette.datastructures import Headers from starlette.requests import Request from starlette.routing import Match, Mount from starlette.types import ASGIApp, Message @@ -151,6 +152,10 @@ async def wrapped_send(message) -> None: _mocked_receive = None _request_receive = None + # begin the transaction before capturing the body to get that time accounted + trace_parent = TraceParent.from_headers(dict(Headers(scope=scope))) + self.client.begin_transaction("request", trace_parent=trace_parent) + if self.client.config.capture_body != "off": # When we consume the body from receive, we replace the streaming @@ -234,9 +239,6 @@ async def _request_started(self, request: Request) -> None: if self.client.config.capture_body != "off": await get_body(request) - trace_parent = TraceParent.from_headers(dict(request.headers)) - self.client.begin_transaction("request", trace_parent=trace_parent) - await set_context(lambda: get_data_from_request(request, self.client.config, constants.TRANSACTION), "request") transaction_name = self.get_route_name(request) or request.url.path elasticapm.set_transaction_name("{} {}".format(request.method, transaction_name), override=False) From bc90dc56dd853ca221e801d976e4d063dfc84b8d Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 16 Feb 2024 12:11:37 +0100 Subject: [PATCH 013/103] tests: make urlllib3 transport tests more robust against local env (#1969) When mocking os.environ pass clear=True to avoid getting host configurations. Fix #1968 --- tests/transports/test_urllib3.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/transports/test_urllib3.py b/tests/transports/test_urllib3.py index 42a21c1e9..b24408e54 100644 --- a/tests/transports/test_urllib3.py +++ b/tests/transports/test_urllib3.py @@ -115,38 +115,46 @@ def test_generic_error(mock_urlopen, elasticapm_client): def test_http_proxy_environment_variable(elasticapm_client): - with mock.patch.dict("os.environ", {"HTTP_PROXY": "http://example.com"}): + with mock.patch.dict("os.environ", {"HTTP_PROXY": "http://example.com"}, clear=True): transport = Transport("http://localhost:9999", client=elasticapm_client) assert isinstance(transport.http, urllib3.ProxyManager) def test_https_proxy_environment_variable(elasticapm_client): - with mock.patch.dict("os.environ", {"HTTPS_PROXY": "https://example.com"}): + with mock.patch.dict( + "os.environ", + { + "HTTPS_PROXY": "https://example.com", + }, + clear=True, + ): transport = Transport("http://localhost:9999", client=elasticapm_client) assert isinstance(transport.http, urllib3.poolmanager.ProxyManager) def test_https_proxy_environment_variable_is_preferred(elasticapm_client): - with mock.patch.dict("os.environ", {"https_proxy": "https://example.com", "HTTP_PROXY": "http://example.com"}): + with mock.patch.dict( + "os.environ", {"https_proxy": "https://example.com", "HTTP_PROXY": "http://example.com"}, clear=True + ): transport = Transport("http://localhost:9999", client=elasticapm_client) assert isinstance(transport.http, urllib3.poolmanager.ProxyManager) assert transport.http.proxy.scheme == "https" def test_no_proxy_star(elasticapm_client): - with mock.patch.dict("os.environ", {"HTTPS_PROXY": "https://example.com", "NO_PROXY": "*"}): + with mock.patch.dict("os.environ", {"HTTPS_PROXY": "https://example.com", "NO_PROXY": "*"}, clear=True): transport = Transport("http://localhost:9999", client=elasticapm_client) assert not isinstance(transport.http, urllib3.poolmanager.ProxyManager) def test_no_proxy_host(elasticapm_client): - with mock.patch.dict("os.environ", {"HTTPS_PROXY": "https://example.com", "NO_PROXY": "localhost"}): + with mock.patch.dict("os.environ", {"HTTPS_PROXY": "https://example.com", "NO_PROXY": "localhost"}, clear=True): transport = Transport("http://localhost:9999", client=elasticapm_client) assert not isinstance(transport.http, urllib3.poolmanager.ProxyManager) def test_no_proxy_all(elasticapm_client): - with mock.patch.dict("os.environ", {"HTTPS_PROXY": "https://example.com", "NO_PROXY": "*"}): + with mock.patch.dict("os.environ", {"HTTPS_PROXY": "https://example.com", "NO_PROXY": "*"}, clear=True): transport = Transport("http://localhost:9999", client=elasticapm_client) assert not isinstance(transport.http, urllib3.poolmanager.ProxyManager) From 8915ec331a6084dbcff0a1e765bd450ae6e3906e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Feb 2024 16:36:30 +0100 Subject: [PATCH 014/103] Bump the github-actions group with 9 updates (#1971) * Bump the github-actions group with 9 updates | Package | From | To | | --- | --- | --- | | [actions/checkout](https://github.com/actions/checkout) | `3` | `4` | | [actions/setup-python](https://github.com/actions/setup-python) | `3` | `5` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `3` | `4` | | [elastic/get-user-teams-membership](https://github.com/elastic/get-user-teams-membership) | `1.0.4` | `1.1.0` | | [pre-commit/action](https://github.com/pre-commit/action) | `3.0.0` | `3.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `3` | `4` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.8.7` | `1.8.11` | | [hashicorp/vault-action](https://github.com/hashicorp/vault-action) | `2.7.2` | `2.8.0` | | [geekyeggo/delete-artifact](https://github.com/geekyeggo/delete-artifact) | `2.0.0` | `4.1.0` | --- .github/workflows/build-distribution.yml | 6 +++--- .github/workflows/labeler.yml | 2 +- .github/workflows/packages.yml | 6 +++--- .github/workflows/pre-commit.yml | 6 +++--- .github/workflows/release.yml | 22 +++++++++++----------- .github/workflows/run-matrix.yml | 2 +- .github/workflows/snapshoty.yml | 4 ++-- .github/workflows/test.yml | 8 ++++---- .github/workflows/updatecli.yml | 2 +- 9 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-distribution.yml b/.github/workflows/build-distribution.yml index 986632acd..8b728c3bf 100644 --- a/.github/workflows/build-distribution.yml +++ b/.github/workflows/build-distribution.yml @@ -7,13 +7,13 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Build lambda layer zip run: ./dev-utils/make-distribution.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: build-distribution path: ./build/ diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index df219658c..377caaa5c 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -17,7 +17,7 @@ jobs: configuration-path: .github/labeler-config.yml enable-versioned-regex: 0 - name: Check team membership for user - uses: elastic/get-user-teams-membership@v1.0.4 + uses: elastic/get-user-teams-membership@1.1.0 id: checkUserMember with: username: ${{ github.actor }} diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 148110c7f..af485c455 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -17,8 +17,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install wheel @@ -28,7 +28,7 @@ jobs: - name: Building source distribution run: python setup.py sdist - name: Upload Packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: packages path: | diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index c2f7e71fc..65947d33b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,6 +9,6 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.0 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 03a77ce47..0e297bb95 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,13 +24,13 @@ jobs: permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: name: packages path: dist - name: Upload - uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0 + uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf with: repository-url: https://upload.pypi.org/legacy/ @@ -42,8 +42,8 @@ jobs: - build-distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: hashicorp/vault-action@v2.7.2 + - uses: actions/checkout@v4 + - uses: hashicorp/vault-action@v2.8.0 with: url: ${{ secrets.VAULT_ADDR }} method: approle @@ -52,7 +52,7 @@ jobs: secrets: | secret/observability-team/ci/service-account/apm-agent-python access_key_id | AWS_ACCESS_KEY_ID ; secret/observability-team/ci/service-account/apm-agent-python secret_access_key | AWS_SECRET_ACCESS_KEY - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-distribution path: ./build @@ -63,7 +63,7 @@ jobs: VERSION=${VERSION//./-} ELASTIC_LAYER_NAME="elastic-apm-python-${VERSION}" .ci/publish-aws.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: arn-file path: ".arn-file.md" @@ -74,7 +74,7 @@ jobs: - build-distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current with: registry: docker.elastic.co @@ -82,7 +82,7 @@ jobs: url: ${{ secrets.VAULT_ADDR }} roleId: ${{ secrets.VAULT_ROLE_ID }} secretId: ${{ secrets.VAULT_SECRET_ID }} - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-distribution path: ./build @@ -115,8 +115,8 @@ jobs: - publish-lambda-layers runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: name: arn-file - name: Create GitHub Draft Release diff --git a/.github/workflows/run-matrix.yml b/.github/workflows/run-matrix.yml index 811f68dd9..827212527 100644 --- a/.github/workflows/run-matrix.yml +++ b/.github/workflows/run-matrix.yml @@ -18,7 +18,7 @@ jobs: matrix: include: ${{ fromJSON(inputs.include) }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run tests run: ./tests/scripts/docker/run_tests.sh ${{ matrix.version }} ${{ matrix.framework }} env: diff --git a/.github/workflows/snapshoty.yml b/.github/workflows/snapshoty.yml index 3f91e2213..49d1b3423 100644 --- a/.github/workflows/snapshoty.yml +++ b/.github/workflows/snapshoty.yml @@ -21,8 +21,8 @@ jobs: - packages runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: name: packages path: dist diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b57b8a023..4638ab5d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -127,7 +127,7 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.ref }} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.version }} cache: pip @@ -185,7 +185,7 @@ jobs: with: ref: ${{ inputs.ref || github.ref }} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: # Use latest Python, so it understands all syntax. python-version: 3.11 @@ -208,10 +208,10 @@ jobs: python -Im coverage report --fail-under=84 - name: Upload HTML report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: html-coverage-report path: htmlcov - - uses: geekyeggo/delete-artifact@54ab544f12cdb7b71613a16a2b5a37a9ade990af + - uses: geekyeggo/delete-artifact@65041433121f7239077fa20be14c0690f70569de with: name: coverage-reports diff --git a/.github/workflows/updatecli.yml b/.github/workflows/updatecli.yml index 22598200b..4fc00bd71 100644 --- a/.github/workflows/updatecli.yml +++ b/.github/workflows/updatecli.yml @@ -12,7 +12,7 @@ jobs: bump: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current with: vaultUrl: ${{ secrets.VAULT_ADDR }} From 0dee7d5e2513ce87d39f513e4a4b4b7a9060e48f Mon Sep 17 00:00:00 2001 From: Mario Candela Date: Wed, 21 Feb 2024 23:17:35 +0100 Subject: [PATCH 015/103] Fix:Update starlette.asciidoc, incorrect middleware order (#1956) * Update starlette.asciidoc Fix typo * Update starlette.asciidoc --------- Co-authored-by: Colton Myers --- docs/starlette.asciidoc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/starlette.asciidoc b/docs/starlette.asciidoc index 77aaca0d4..941bf6d7a 100644 --- a/docs/starlette.asciidoc +++ b/docs/starlette.asciidoc @@ -42,10 +42,12 @@ app = Starlette() app.add_middleware(ElasticAPM) ---- -WARNING: If you are using any `BaseHTTPMiddleware` middleware, you must add them -*before* the ElasticAPM middleware. This is because `BaseHTTPMiddleware` breaks -`contextvar` propagation, as noted -https://www.starlette.io/middleware/#limitations[here]. +WARNING: `BaseHTTPMiddleware` breaks `contextvar` propagation, as noted +https://www.starlette.io/middleware/#limitations[here]. This means the +ElasticAPM middleware must be above any `BaseHTTPMiddleware` in the final +middleware list. If you're calling `add_middleware` repeatedly, add the +ElasticAPM middleware last. If you're passing in a list of middleware, +ElasticAPM should be first on that list. To configure the agent using initialization arguments: From d5b8c454684d81a38c83adafe4feaa206bd6d452 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 23 Feb 2024 09:58:59 +0100 Subject: [PATCH 016/103] instrumentation/dbapi2: make query scanning for dollar quotes a bit more correct (#1976) Dollar quotes follow the same rules as SQL identifiers so: SQL identifiers and key words must begin with a letter (a-z, but also letters with diacritical marks and non-Latin letters) or an underscore (_). Subsequent characters in an identifier or key word can be letters, underscores, digits (0-9), or dollar signs ($). Given we are not going to write a compliant SQL parser at least handle query parameters that are simple to catch since they have a digit right after the opening $. Refs #1851. --- elasticapm/instrumentation/packages/dbapi2.py | 7 +++++++ tests/instrumentation/dbapi2_tests.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/elasticapm/instrumentation/packages/dbapi2.py b/elasticapm/instrumentation/packages/dbapi2.py index fb49723c2..fa1d0f31e 100644 --- a/elasticapm/instrumentation/packages/dbapi2.py +++ b/elasticapm/instrumentation/packages/dbapi2.py @@ -34,6 +34,7 @@ """ import re +import string import wrapt @@ -85,6 +86,7 @@ def scan(tokens): literal_started = None prev_was_escape = False lexeme = [] + digits = set(string.digits) i = 0 while i < len(tokens): @@ -114,6 +116,11 @@ def scan(tokens): literal_start_idx = i literal_started = token elif token == "$": + # exclude query parameters that have a digit following the dollar + if True and len(tokens) > i + 1 and tokens[i + 1] in digits: + yield i, token + i += 1 + continue # Postgres can use arbitrary characters between two $'s as a # literal separation token, e.g.: $fish$ literal $fish$ # This part will detect that and skip over the literal. diff --git a/tests/instrumentation/dbapi2_tests.py b/tests/instrumentation/dbapi2_tests.py index 3d72b6632..089571715 100644 --- a/tests/instrumentation/dbapi2_tests.py +++ b/tests/instrumentation/dbapi2_tests.py @@ -122,6 +122,20 @@ def test_extract_signature_bytes(): assert actual == expected +def test_extract_signature_pathological(): + # tune for performance testing + multiplier = 10 + values = [] + for chunk in range(multiplier): + i = chunk * 3 + values.append(f" (${1+i}::varchar, ${2+i}::varchar, ${3+i}::varchar), ") + + sql = f"SELECT * FROM (VALUES {''.join(values)})\n" + actual = extract_signature(sql) + expected = "SELECT FROM" + assert actual == expected + + @pytest.mark.parametrize( ["sql", "expected"], [ From 8263d994e86b866de956581876382dcf82493820 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Sat, 24 Feb 2024 09:49:40 +0100 Subject: [PATCH 017/103] Cleanup python2 vestiges (#1977) Remove python2 vestiges --- elasticapm/instrumentation/packages/urllib.py | 5 ++--- elasticapm/utils/__init__.py | 10 ++-------- tests/client/client_tests.py | 6 +----- tests/fixtures.py | 7 +------ tests/instrumentation/urllib_tests.py | 2 +- tests/utils/tests.py | 10 +--------- 6 files changed, 8 insertions(+), 32 deletions(-) diff --git a/elasticapm/instrumentation/packages/urllib.py b/elasticapm/instrumentation/packages/urllib.py index b40932a55..2b0dae16e 100644 --- a/elasticapm/instrumentation/packages/urllib.py +++ b/elasticapm/instrumentation/packages/urllib.py @@ -97,10 +97,9 @@ def call(self, module, method, wrapped, instance, args, kwargs): leaf_span.dist_tracing_propagated = True response = wrapped(*args, **kwargs) if response: - status = getattr(response, "status", None) or response.getcode() # Python 2 compat if span.context: - span.context["http"]["status_code"] = status - span.set_success() if status < 400 else span.set_failure() + span.context["http"]["status_code"] = response.status + span.set_success() if response.status < 400 else span.set_failure() return response def mutate_unsampled_call_args(self, module, method, wrapped, instance, args, kwargs, transaction): diff --git a/elasticapm/utils/__init__.py b/elasticapm/utils/__init__.py index 58a302960..0f7b52c0d 100644 --- a/elasticapm/utils/__init__.py +++ b/elasticapm/utils/__init__.py @@ -33,20 +33,14 @@ import re import socket import urllib.parse -from functools import partial +from functools import partial, partialmethod from types import FunctionType from typing import Pattern from elasticapm.conf import constants from elasticapm.utils import encoding -try: - from functools import partialmethod - - partial_types = (partial, partialmethod) -except ImportError: - # Python 2 - partial_types = (partial,) +partial_types = (partial, partialmethod) default_ports = {"https": 443, "http": 80, "postgresql": 5432, "mysql": 3306, "mssql": 1433} diff --git a/tests/client/client_tests.py b/tests/client/client_tests.py index 6cec88205..af266b710 100644 --- a/tests/client/client_tests.py +++ b/tests/client/client_tests.py @@ -77,11 +77,7 @@ def test_service_info_node_name(elasticapm_client): def test_process_info(elasticapm_client): process_info = elasticapm_client.get_process_info() assert process_info["pid"] == os.getpid() - if hasattr(os, "getppid"): - assert process_info["ppid"] == os.getppid() - else: - # Windows + Python 2.7 - assert process_info["ppid"] is None + assert process_info["ppid"] == os.getppid() assert "argv" not in process_info elasticapm_client.config.update("1", include_process_args=True) with mock.patch.object(sys, "argv", ["a", "b", "c"]): diff --git a/tests/fixtures.py b/tests/fixtures.py index 94e89f961..ddeaa1f5b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -46,6 +46,7 @@ import zlib from collections import defaultdict from typing import Optional +from urllib.request import pathname2url import jsonschema import mock @@ -61,12 +62,6 @@ from elasticapm.transport.http_base import HTTPTransportBase from elasticapm.utils.threading import ThreadManager -try: - from urllib.request import pathname2url -except ImportError: - # Python 2 - from urllib import pathname2url - cur_dir = os.path.dirname(os.path.realpath(__file__)) ERRORS_SCHEMA = os.path.join(cur_dir, "upstream", "json-specs", "error.json") diff --git a/tests/instrumentation/urllib_tests.py b/tests/instrumentation/urllib_tests.py index 3f2796483..fbf5fa44f 100644 --- a/tests/instrumentation/urllib_tests.py +++ b/tests/instrumentation/urllib_tests.py @@ -114,7 +114,7 @@ def test_urllib_error(instrument, elasticapm_client, waiting_httpserver, status_ @mock.patch(request_method) @mock.patch(getresponse_method) def test_urllib_standard_port(mock_getresponse, mock_request, instrument, elasticapm_client): - # "code" is needed for Python 3, "status" for Python 2 + # Python internally used both "code" and "status" mock_getresponse.return_value = mock.Mock(code=200, status=200) url = "http://example.com/" diff --git a/tests/utils/tests.py b/tests/utils/tests.py index 5f073862d..bd09eef26 100644 --- a/tests/utils/tests.py +++ b/tests/utils/tests.py @@ -30,7 +30,7 @@ import os import socket -from functools import partial +from functools import partial, partialmethod import pytest @@ -48,12 +48,6 @@ ) from elasticapm.utils.deprecation import deprecated -try: - from functools import partialmethod -except ImportError: - # Python 2 - partialmethod = None - @deprecated("alternative") def deprecated_function(): @@ -164,7 +158,6 @@ def x(x): assert "partial(tests.utils.tests.x)" == get_name_from_func(p) -@pytest.mark.skipif(partialmethod is None, reason="partialmethod not available on Python 2") def test_get_name_from_func_partialmethod_unbound(): class X(object): def x(self, x): @@ -175,7 +168,6 @@ def x(self, x): assert "partial(tests.utils.tests.x)" == get_name_from_func(X.p) -@pytest.mark.skipif(partialmethod is None, reason="partialmethod not available on Python 2") def test_get_name_from_func_partialmethod_bound(): class X(object): def x(self, x): From 1cfe84d0d2c14560b1752b76fb45b414c22318b9 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Sat, 24 Feb 2024 09:50:20 +0100 Subject: [PATCH 018/103] ci: re-enable more python 3.12 testing (#1978) --- .ci/.matrix_exclude.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.ci/.matrix_exclude.yml b/.ci/.matrix_exclude.yml index d7d05f2af..5eb71a09f 100644 --- a/.ci/.matrix_exclude.yml +++ b/.ci/.matrix_exclude.yml @@ -263,29 +263,9 @@ exclude: - VERSION: python-3.7 FRAMEWORK: celery-4-flask-1.0 # TODO py3.12 - - VERSION: python-3.12 - FRAMEWORK: pymssql-newest # no wheels available yet - - VERSION: python-3.12 - FRAMEWORK: aiohttp-newest # no wheels available yet - - VERSION: python-3.12 - FRAMEWORK: elasticsearch-7 # relies on aiohttp - - VERSION: python-3.12 - FRAMEWORK: elasticsearch-8 # relies on aiohttp - - VERSION: python-3.12 - FRAMEWORK: aiobotocore-newest # relies on aiohttp - VERSION: python-3.12 FRAMEWORK: sanic-20.12 # no wheels available yet - - VERSION: python-3.12 - FRAMEWORK: sanic-newest # no wheels available yet - VERSION: python-3.12 FRAMEWORK: kafka-python-newest # https://github.com/dpkp/kafka-python/pull/2376 - - VERSION: python-3.12 - FRAMEWORK: pyodbc-newest # error on wheel - VERSION: python-3.12 FRAMEWORK: cassandra-newest # c extension issue - - VERSION: python-3.12 - FRAMEWORK: starlette-newest # waiting for 3.12.2 for this fix: https://github.com/python/cpython/pull/111221 - - VERSION: python-3.12 - FRAMEWORK: starlette-0.14 # waiting for 3.12.2 for this fix: https://github.com/python/cpython/pull/111221 - - VERSION: python-3.12 - FRAMEWORK: starlette-0.13 # waiting for 3.12.2 for this fix: https://github.com/python/cpython/pull/111221 From a7ba8c7f84b5d5734f81646117025a41c40b95a2 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 5 Mar 2024 09:23:25 +0100 Subject: [PATCH 019/103] Remove hound integration (#1983) We have already pre-commit running on CI and it doesn't look to be running at the moment. --- .hound.yml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .hound.yml diff --git a/.hound.yml b/.hound.yml deleted file mode 100644 index 0745a960a..000000000 --- a/.hound.yml +++ /dev/null @@ -1,5 +0,0 @@ -flake8: - enabled: true - config_file: .flake8 - -fail_on_violations: true From d9c53fcae9a45e671bbee27c7766936906159a8c Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 5 Mar 2024 09:28:18 +0100 Subject: [PATCH 020/103] contrib/serverless/aws: normalize headers with v1 event format (#1982) If the event has been generated from elb or api gateway and is in the v1 format normalize headers even if they already should per https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html Event discrimination logic courtesy of nodejs apm client. Fix #1980 --- elasticapm/contrib/serverless/aws.py | 20 ++++++++++++++++- .../contrib/serverless/aws_elb_test_data.json | 1 + tests/contrib/serverless/aws_tests.py | 22 ++++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/elasticapm/contrib/serverless/aws.py b/elasticapm/contrib/serverless/aws.py index 26f37bdfb..9f5f7b133 100644 --- a/elasticapm/contrib/serverless/aws.py +++ b/elasticapm/contrib/serverless/aws.py @@ -135,6 +135,18 @@ def prep_kwargs(kwargs=None): return kwargs +def should_normalize_headers(event: dict) -> bool: + """ + Helper to decide if we should normalize headers or not depending on the event + + Even if the documentation says that headers are lowercased it's not always the case for format version 1.0 + https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html + """ + + request_context = event.get("requestContext", {}) + return ("elb" in request_context or "requestId" in request_context) and "http" not in request_context + + class _lambda_transaction(object): """ Context manager for creating transactions around AWS Lambda functions. @@ -162,7 +174,13 @@ def __enter__(self): # service like Step Functions, and is unlikely to be standardized # in any way. We just have to rely on our defaults in this case. self.event = {} - trace_parent = TraceParent.from_headers(self.event.get("headers") or {}) + + headers = self.event.get("headers") or {} + if headers and should_normalize_headers(self.event): + normalized_headers = {k.lower(): v for k, v in headers.items()} + else: + normalized_headers = headers + trace_parent = TraceParent.from_headers(normalized_headers) global COLD_START cold_start = COLD_START diff --git a/tests/contrib/serverless/aws_elb_test_data.json b/tests/contrib/serverless/aws_elb_test_data.json index 87e05ac85..79b4dc6dd 100644 --- a/tests/contrib/serverless/aws_elb_test_data.json +++ b/tests/contrib/serverless/aws_elb_test_data.json @@ -15,6 +15,7 @@ "connection": "Keep-Alive", "host": "blabla.com", "user-agent": "Apache-HttpClient/4.5.13 (Java/11.0.15)", + "TraceParent": "00-12345678901234567890123456789012-1234567890123456-01", "x-amzn-trace-id": "Root=1-xxxxxxxxxxxxxx", "x-forwarded-for": "199.99.99.999", "x-forwarded-port": "443", diff --git a/tests/contrib/serverless/aws_tests.py b/tests/contrib/serverless/aws_tests.py index 9f4a7253f..df062a378 100644 --- a/tests/contrib/serverless/aws_tests.py +++ b/tests/contrib/serverless/aws_tests.py @@ -36,7 +36,12 @@ from elasticapm import capture_span from elasticapm.conf import constants -from elasticapm.contrib.serverless.aws import capture_serverless, get_data_from_request, get_data_from_response +from elasticapm.contrib.serverless.aws import ( + capture_serverless, + get_data_from_request, + get_data_from_response, + should_normalize_headers, +) @pytest.fixture @@ -300,6 +305,7 @@ def test_func(event, context): assert transaction["context"]["request"]["headers"] assert transaction["context"]["response"]["status_code"] == 200 assert transaction["context"]["service"]["origin"]["name"] == "lambda-279XGJDqGZ5rsrHC2Fjr" + assert transaction["trace_id"] == "12345678901234567890123456789012" def test_capture_serverless_s3(event_s3, context, elasticapm_client): @@ -477,3 +483,17 @@ def test_func(event, context): test_func(event_api2, context) assert len(elasticapm_client.events[constants.TRANSACTION]) == 1 + + +def test_should_normalize_headers_true(event_api, event_elb): + assert should_normalize_headers(event_api) is True + assert should_normalize_headers(event_elb) is True + + +def test_should_normalize_headers_false(event_api2, event_lurl, event_s3, event_s3_batch, event_sqs, event_sns): + assert should_normalize_headers(event_api2) is False + assert should_normalize_headers(event_lurl) is False + assert should_normalize_headers(event_s3) is False + assert should_normalize_headers(event_s3_batch) is False + assert should_normalize_headers(event_sqs) is False + assert should_normalize_headers(event_sns) is False From 22db8e1d8cc8b49a85479c7e9cadc7ba3fd9081c Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 6 Mar 2024 17:22:27 +0100 Subject: [PATCH 021/103] update CHANGELOG and bump version to 6.21.0 (#1986) --- CHANGELOG.asciidoc | 16 +++++++++++++++- elasticapm/version.py | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index fb87b1107..c94edf53a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -5,7 +5,7 @@ endif::[] //// [[release-notes-x.x.x]] -==== x.x.x - YYYY/MM/DD +==== x.x.x - YYYY-MM-DD [float] ===== Breaking changes @@ -32,6 +32,20 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.21.0]] +==== 6.21.0 - 2024-03-06 + +[float] +===== Bug fixes + +* Fix starlette middleware setup without client argument {pull}1952[#1952] +* Fix blocking of gRPC stream-to-stream requests {pull}1967[#1967] +* Always take into account body reading time for starlette requests {pull}1970[#1970] +* Make urllib3 transport tests more robust against local env {pull}1969[#1969] +* Clarify starlette integration documentation {pull}1956[#1956] +* Make dbapi2 query scanning for dollar quotes a bit more correct {pull}1976[#1976] +* Normalize headers in AWS Lambda integration on API Gateway v1 requests {pull}1982[#1982] + [[release-notes-6.20.0]] ==== 6.20.0 - 2024-01-10 diff --git a/elasticapm/version.py b/elasticapm/version.py index ea64e853b..a48ac9368 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 20, 0) +__version__ = (6, 21, 0) VERSION = ".".join(map(str, __version__)) From 033ea0ce0459c396bb901e20c98bb4a9c9b970fa Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 7 Mar 2024 10:09:37 +0100 Subject: [PATCH 022/103] ci: add missing full-matrix input when using test workflow in release (#1990) Should fix the following error when tagging a release: The workflow is not valid. .github/workflows/release.yml (Line: 13, Col: 11): Input full-matrix is required, but not provided while calling. --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e297bb95..8d22d37aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,8 @@ permissions: jobs: test: uses: ./.github/workflows/test.yml + with: + full-matrix: true packages: uses: ./.github/workflows/packages.yml From d371d23aac5e364a2f025c5f1bafc0b1be7916ba Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 7 Mar 2024 16:59:41 +0100 Subject: [PATCH 023/103] Update CHANGELOG and bump version to 6.21.1 (#1992) --- CHANGELOG.asciidoc | 8 ++++++++ elasticapm/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index c94edf53a..b8ad0d90c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -32,6 +32,14 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.21.1]] +==== 6.21.1 - 2024-03-07 + +[float] +===== Bug fixes + +* Fix CI release workflow {pull}1990[#1990] + [[release-notes-6.21.0]] ==== 6.21.0 - 2024-03-06 diff --git a/elasticapm/version.py b/elasticapm/version.py index a48ac9368..7cba6757b 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 21, 0) +__version__ = (6, 21, 1) VERSION = ".".join(map(str, __version__)) From 57329c090f5b27b2609bbf2bf37f78142b533b10 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 7 Mar 2024 17:53:10 +0100 Subject: [PATCH 024/103] ci: revert to actions/upload-artifact@v3 in build-distribution workflow (#1993) It looks like since v4 only one upload for the same dir will work. See: https://github.com/actions/upload-artifact/issues/478#issuecomment-1860038048 In the meantime we sort this out revert to v3. Should fix: https://github.com/elastic/apm-agent-python/actions/runs/8191161913/job/22399752533 --- .github/workflows/build-distribution.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-distribution.yml b/.github/workflows/build-distribution.yml index 8b728c3bf..fd3e11ed7 100644 --- a/.github/workflows/build-distribution.yml +++ b/.github/workflows/build-distribution.yml @@ -13,7 +13,7 @@ jobs: python-version: "3.10" - name: Build lambda layer zip run: ./dev-utils/make-distribution.sh - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v3 with: name: build-distribution path: ./build/ From a735a00d4d6b6fd2b4cdbad0e304e42df36513e9 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 7 Mar 2024 19:19:30 +0100 Subject: [PATCH 025/103] update CHANGELOG and bump version to 6.21.2 (#1995) --- CHANGELOG.asciidoc | 8 ++++++++ elasticapm/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index b8ad0d90c..956f90d9a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -32,6 +32,14 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.21.2]] +==== 6.21.2 - 2024-03-07 + +[float] +===== Bug fixes + +* Fix artifacts upload in CI build-distribution workflow {pull}1993[#1993] + [[release-notes-6.21.1]] ==== 6.21.1 - 2024-03-07 diff --git a/elasticapm/version.py b/elasticapm/version.py index 7cba6757b..38a43e781 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 21, 1) +__version__ = (6, 21, 2) VERSION = ".".join(map(str, __version__)) From da1bec493cfadbcedc210e9d1979c257c02d3dfc Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 8 Mar 2024 11:38:18 +0100 Subject: [PATCH 026/103] ci: switch build-distribution downloadArtifact to v3 (#1996) Since v4 can't find them if uploaded with uploadArtifact v3. --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8d22d37aa..a2a48b62b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,7 +54,7 @@ jobs: secrets: | secret/observability-team/ci/service-account/apm-agent-python access_key_id | AWS_ACCESS_KEY_ID ; secret/observability-team/ci/service-account/apm-agent-python secret_access_key | AWS_SECRET_ACCESS_KEY - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v3 with: name: build-distribution path: ./build @@ -84,7 +84,7 @@ jobs: url: ${{ secrets.VAULT_ADDR }} roleId: ${{ secrets.VAULT_ROLE_ID }} secretId: ${{ secrets.VAULT_SECRET_ID }} - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v3 with: name: build-distribution path: ./build From afe32ca0b0a987d8087f76486455ed959c0f893a Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 8 Mar 2024 12:00:32 +0100 Subject: [PATCH 027/103] update CHANGELOG and bump version to 6.21.3 (#1997) --- CHANGELOG.asciidoc | 8 ++++++++ elasticapm/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 956f90d9a..57a6cb95f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -32,6 +32,14 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.21.3]] +==== 6.21.3 - 2024-03-08 + +[float] +===== Bug fixes + +* Fix artifacts download in CI workflows {pull}1996[#1996] + [[release-notes-6.21.2]] ==== 6.21.2 - 2024-03-07 diff --git a/elasticapm/version.py b/elasticapm/version.py index 38a43e781..6da6c370d 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 21, 2) +__version__ = (6, 21, 3) VERSION = ".".join(map(str, __version__)) From 433e4d9c0c3598a9a1f4420c96b5a73cdb8b5c73 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 13 Mar 2024 10:40:46 +0100 Subject: [PATCH 028/103] instrumentation/urllib3: fix urllib3 2.0.1+ crash with many args (#2002) In c1dd69e0a5b9d1ec020666a97e0b7cb788b86c4f we changed the logic of update_headers taking into account a new body arg before the one for headers. The problem is that HTTPConnectionPool.urlopen did not change at all, only HTTPConnectionPool.request did so the old login update_headers was fine. Fix #1928 --- elasticapm/instrumentation/packages/urllib3.py | 7 +------ tests/instrumentation/urllib3_tests.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/elasticapm/instrumentation/packages/urllib3.py b/elasticapm/instrumentation/packages/urllib3.py index cc7206e83..93d9c3392 100644 --- a/elasticapm/instrumentation/packages/urllib3.py +++ b/elasticapm/instrumentation/packages/urllib3.py @@ -61,12 +61,7 @@ def update_headers(args, kwargs, instance, transaction, trace_parent): :param trace_parent: the TraceParent object :return: an (args, kwargs) tuple """ - from urllib3._version import __version__ as urllib3_version - - if urllib3_version.startswith("2") and len(args) >= 5 and args[4]: - headers = args[4].copy() - args = tuple(itertools.chain((args[:4]), (headers,), args[5:])) - elif len(args) >= 4 and args[3]: + if len(args) >= 4 and args[3]: headers = args[3].copy() args = tuple(itertools.chain((args[:3]), (headers,), args[4:])) elif "headers" in kwargs and kwargs["headers"]: diff --git a/tests/instrumentation/urllib3_tests.py b/tests/instrumentation/urllib3_tests.py index 8cc21ceb0..1fa03fa43 100644 --- a/tests/instrumentation/urllib3_tests.py +++ b/tests/instrumentation/urllib3_tests.py @@ -294,3 +294,20 @@ def test_instance_headers_are_respected( assert "kwargs" in request_headers if instance_headers and not (header_arg or header_kwarg): assert "instance" in request_headers + + +def test_connection_pool_urlopen_does_not_crash_with_many_args(instrument, elasticapm_client, waiting_httpserver): + """Mimics ConnectionPool.urlopen error path with broken connection, see #1928""" + waiting_httpserver.serve_content("") + url = waiting_httpserver.url + "/hello_world" + parsed_url = urllib.parse.urlparse(url) + pool = urllib3.HTTPConnectionPool( + parsed_url.hostname, + parsed_url.port, + maxsize=1, + block=True, + ) + retry = urllib3.util.Retry(10) + elasticapm_client.begin_transaction("transaction") + r = pool.urlopen("GET", url, None, {"args": "true"}, retry, False, False) + assert r.status == 200 From 93c362d814d066433c68ad702735ea5371028543 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 13 Mar 2024 17:58:17 +0100 Subject: [PATCH 029/103] setup: take a post version from environment variable (#2000) Make it possible to add a post version (per PEP-440) to the current agent version at build time. This will be useful because we want to continuously build the agent on CI and avoid version conflicts when publishing to test pypi. So when returning the agent version add anything that has been passed in ELASTIC_CI_POST_VERSION environment variable as post version. Refs #1994 --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 23ebec33e..a88cdeb5b 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,11 @@ def get_version(): for line in version_file: if line.startswith("__version__"): version_tuple = ast.literal_eval(line.split(" = ")[1]) - return ".".join(map(str, version_tuple)) + version_str = ".".join(map(str, version_tuple)) + post_version = os.getenv("ELASTIC_CI_POST_VERSION") + if post_version: + return f"{version_str}.post{post_version}" + return version_str return "unknown" From 38fb16acd12ff9ce5abf00dac4edf34e831a93c4 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 13 Mar 2024 18:01:17 +0100 Subject: [PATCH 030/103] Deprecate Python < 3.2 logging code and and remove documentation (#1984) * docs: remove Python < 3.2 specific documentation * handlers/logging: deprecate Python<3.2 LoggingFilter Since we don't support Python < 3.6 there's not reason to keep code for Python versions older than that. * Update elasticapm/handlers/logging.py Co-authored-by: Colton Myers --------- Co-authored-by: Colton Myers --- docs/logging.asciidoc | 21 +-------------------- elasticapm/handlers/logging.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/docs/logging.asciidoc b/docs/logging.asciidoc index 943de8f64..8f51edd50 100644 --- a/docs/logging.asciidoc +++ b/docs/logging.asciidoc @@ -36,7 +36,7 @@ as well as http://www.structlog.org/en/stable/[`structlog`]. [[logging]] ===== `logging` -For Python 3.2+, we use https://docs.python.org/3/library/logging.html#logging.setLogRecordFactory[`logging.setLogRecordFactory()`] +We use https://docs.python.org/3/library/logging.html#logging.setLogRecordFactory[`logging.setLogRecordFactory()`] to decorate the default LogRecordFactory to automatically add new attributes to each LogRecord object: @@ -51,25 +51,6 @@ You can disable this automatic behavior by using the <> setting in your configuration. -For Python versions <3.2, we also provide a -https://docs.python.org/3/library/logging.html#filter-objects[filter] which will -add the same new attributes to any filtered `LogRecord`: - -[source,python] ----- -import logging -from elasticapm.handlers.logging import LoggingFilter - -console = logging.StreamHandler() -console.addFilter(LoggingFilter()) -# add the handler to the root logger -logging.getLogger("").addHandler(console) ----- - -NOTE: Because https://docs.python.org/3/library/logging.html#filter-objects[filters -are not propagated to descendent loggers], you should add the filter to each of -your log handlers, as handlers are propagated, along with their attached filters. - [float] [[structlog]] ===== `structlog` diff --git a/elasticapm/handlers/logging.py b/elasticapm/handlers/logging.py index 4407f0f87..ed4db87ac 100644 --- a/elasticapm/handlers/logging.py +++ b/elasticapm/handlers/logging.py @@ -194,6 +194,16 @@ class LoggingFilter(logging.Filter): automatically. """ + def __init__(self, name=""): + super().__init__(name=name) + warnings.warn( + "The LoggingFilter is deprecated and will be removed in v7.0 of " + "the agent. On Python 3.2+, by default we add a LogRecordFactory to " + "your root logger automatically" + "https://www.elastic.co/guide/en/apm/agent/python/current/logs.html", + PendingDeprecationWarning, + ) + def filter(self, record): """ Add elasticapm attributes to `record`. From 1d714a673fdef69f03322b36c5601a1b761bc359 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 13 Mar 2024 18:08:27 +0100 Subject: [PATCH 031/103] ci: run the release when merges to main (#1998) * ci: run the release when merges to main * avoid wrong docker-tag name * use the opposite * fix yaml * enable test, slack message conditional and fix names * Update .github/workflows/release.yml * skip test for branches in the release workflow * workaround * fix * Update .github/workflows/release.yml * support releases in test.pypi for commits on main * set environment variable in the reusable workflow instead env propagation between reusable workflow is not supported * setup: take a post version from environment variable * Update .github/workflows/release.yml --------- Co-authored-by: Riccardo Magliocchetti --- .github/workflows/packages.yml | 4 +++ .github/workflows/release.yml | 39 +++++++++++++++++++++-------- .github/workflows/test-release.yml | 40 ++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/test-release.yml diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index af485c455..f2d9a4f83 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -13,6 +13,10 @@ on: - '**/*.md' - '**/*.asciidoc' +# Override the version if there is no tag release. +env: + ELASTIC_CI_POST_VERSION: ${{ startsWith(github.ref, 'refs/tags') && '' || github.run_id }} + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2a48b62b..b1d0dff48 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,15 +4,18 @@ on: push: tags: - "v*.*.*" + branches: + - main permissions: contents: read jobs: test: - uses: ./.github/workflows/test.yml + uses: ./.github/workflows/test-release.yml with: full-matrix: true + enabled: ${{ startsWith(github.ref, 'refs/tags') }} packages: uses: ./.github/workflows/packages.yml @@ -31,10 +34,16 @@ jobs: with: name: packages path: dist - - name: Upload + - name: Upload pypi.org + if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf with: repository-url: https://upload.pypi.org/legacy/ + - name: Upload test.pypi.org + if: ${{ ! startsWith(github.ref, 'refs/tags') }} + uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf + with: + repository-url: https://test.pypi.org/legacy/ build-distribution: uses: ./.github/workflows/build-distribution.yml @@ -59,6 +68,7 @@ jobs: name: build-distribution path: ./build - name: Publish lambda layers to AWS + if: startsWith(github.ref, 'refs/tags') run: | # Convert v1.2.3 to ver-1-2-3 VERSION=${GITHUB_REF_NAME/v/ver-} @@ -66,6 +76,7 @@ jobs: ELASTIC_LAYER_NAME="elastic-apm-python-${VERSION}" .ci/publish-aws.sh - uses: actions/upload-artifact@v4 + if: startsWith(github.ref, 'refs/tags') with: name: arn-file path: ".arn-file.md" @@ -75,6 +86,8 @@ jobs: needs: - build-distribution runs-on: ubuntu-latest + env: + DOCKER_IMAGE_NAME: docker.elastic.co/observability/apm-agent-python steps: - uses: actions/checkout@v4 - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current @@ -91,30 +104,35 @@ jobs: - id: setup-docker name: Set up docker variables run: |- - # version without v prefix (e.g. 1.2.3) - echo "tag=${GITHUB_REF_NAME/v/}" >> "${GITHUB_OUTPUT}" - echo "name=docker.elastic.co/observability/apm-agent-python" >> "${GITHUB_OUTPUT}" + if [ "${{ startsWith(github.ref, 'refs/tags') }}" == "false" ] ; then + # for testing purposes + echo "tag=test" >> "${GITHUB_OUTPUT}" + else + # version without v prefix (e.g. 1.2.3) + echo "tag=${GITHUB_REF_NAME/v/}" >> "${GITHUB_OUTPUT}" + fi - name: Docker build run: >- docker build - -t ${{ steps.setup-docker.outputs.name }}:${{ steps.setup-docker.outputs.tag }} + -t ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} --build-arg AGENT_DIR=./build/dist/package/python . - name: Docker retag run: >- docker tag - ${{ steps.setup-docker.outputs.name }}:${{ steps.setup-docker.outputs.tag }} - ${{ steps.setup-docker.outputs.name }}:latest + ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} + ${{ env.DOCKER_IMAGE_NAME }}:latest - name: Docker push + if: startsWith(github.ref, 'refs/tags') run: |- - docker push ${{ steps.setup-docker.outputs.name }}:${{ steps.setup-docker.outputs.tag }} - docker push ${{ steps.setup-docker.outputs.name }}:latest + docker push --all-tags ${{ env.DOCKER_IMAGE_NAME }} github-draft: permissions: contents: write needs: - publish-lambda-layers + if: startsWith(github.ref, 'refs/tags') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -145,6 +163,7 @@ jobs: with: needs: ${{ toJSON(needs) }} - uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current + if: startsWith(github.ref, 'refs/tags') with: status: ${{ steps.check.outputs.status }} vaultUrl: ${{ secrets.VAULT_ADDR }} diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml new file mode 100644 index 000000000..c873f9eb7 --- /dev/null +++ b/.github/workflows/test-release.yml @@ -0,0 +1,40 @@ +name: test-release + +on: + workflow_call: + inputs: + full-matrix: + description: "Run the full matrix" + required: true + type: boolean + ref: + description: "The git ref of elastic/apm-agent-python to run test workflow from." + required: false + type: string + enabled: + description: "Whether to run the workfow" + required: true + type: boolean + workflow_dispatch: + inputs: + full-matrix: + description: "Run the full matrix" + required: true + type: boolean + enabled: + description: "Whether to run the workfow" + required: true + type: boolean + +jobs: + test: + if: ${{ inputs.enabled }} + uses: ./.github/workflows/test.yml + with: + full-matrix: ${{ inputs.full-matrix }} + + run-if-disabled: + if: ${{ ! inputs.enabled }} + runs-on: ubuntu-latest + steps: + - run: echo "do something to help with the reusable workflows with needs" From 9f384de55ea7f992800cb6c276662183a150df64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 09:55:47 +0100 Subject: [PATCH 032/103] Bump the github-actions group with 2 updates (#1999) Bumps the github-actions group with 2 updates: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) and [hashicorp/vault-action](https://github.com/hashicorp/vault-action). Updates `pypa/gh-action-pypi-publish` from 1.8.11 to 1.8.14 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf...81e9d935c883d0b210363ab89cf05f3894778450) Updates `hashicorp/vault-action` from 2.8.0 to 3.0.0 - [Release notes](https://github.com/hashicorp/vault-action/releases) - [Changelog](https://github.com/hashicorp/vault-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/vault-action/compare/v2.8.0...v3.0.0) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: hashicorp/vault-action dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b1d0dff48..5f91014b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,12 +36,12 @@ jobs: path: dist - name: Upload pypi.org if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf + uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 with: repository-url: https://upload.pypi.org/legacy/ - name: Upload test.pypi.org if: ${{ ! startsWith(github.ref, 'refs/tags') }} - uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf + uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 with: repository-url: https://test.pypi.org/legacy/ @@ -54,7 +54,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: hashicorp/vault-action@v2.8.0 + - uses: hashicorp/vault-action@v3.0.0 with: url: ${{ secrets.VAULT_ADDR }} method: approle From 845382d5fd1cd83238163938e3aae95250a12e4c Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 14 Mar 2024 14:50:57 +0100 Subject: [PATCH 033/103] ci: update issue labeler (#2003) * ci: update issue labeler workflow Stop using the apm group and custom github action to check if issues are from internal members or not. While at it remove posting internal pull requests on the old APM agent board. Fixes #2001 * Update .github/workflows/labeler.yml Co-authored-by: Jan Calanog --------- Co-authored-by: Jan Calanog --- .github/community-label.yml | 5 ---- .github/labeler-config.yml | 3 --- .github/workflows/labeler.yml | 49 +++++++++++++---------------------- 3 files changed, 18 insertions(+), 39 deletions(-) delete mode 100644 .github/community-label.yml delete mode 100644 .github/labeler-config.yml diff --git a/.github/community-label.yml b/.github/community-label.yml deleted file mode 100644 index 8872df2d5..000000000 --- a/.github/community-label.yml +++ /dev/null @@ -1,5 +0,0 @@ -# add 'community' label to all new issues and PRs created by the community -community: - - '.*' -triage: - - '.*' \ No newline at end of file diff --git a/.github/labeler-config.yml b/.github/labeler-config.yml deleted file mode 100644 index a1e4dbc29..000000000 --- a/.github/labeler-config.yml +++ /dev/null @@ -1,3 +0,0 @@ -# add 'agent-python' label to all new issues -agent-python: - - '.*' diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 377caaa5c..564cc7d31 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -4,43 +4,30 @@ on: types: [opened] pull_request_target: types: [opened] -env: - MY_GITHUB_TOKEN: ${{ secrets.APM_TECH_USER_TOKEN }} + +# 'issues: write' for https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#add-labels-to-an-issue +permissions: + contents: read + issues: write + pull-requests: write + jobs: triage: runs-on: ubuntu-latest steps: - name: Add agent-python label - uses: AlexanderWert/issue-labeler@v2.3 + uses: actions-ecosystem/action-add-labels@v1 with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" - configuration-path: .github/labeler-config.yml - enable-versioned-regex: 0 - - name: Check team membership for user - uses: elastic/get-user-teams-membership@1.1.0 - id: checkUserMember + labels: agent-python + - id: is_elastic_member + uses: elastic/apm-pipeline-library/.github/actions/is-member-elastic-org@current with: username: ${{ github.actor }} - team: 'apm' - usernamesToExclude: | - apmmachine - dependabot - GITHUB_TOKEN: ${{ secrets.APM_TECH_USER_TOKEN }} - - name: Show team membership - run: | - echo "::debug::isTeamMember: ${{ steps.checkUserMember.outputs.isTeamMember }}" - echo "::debug::isExcluded: ${{ steps.checkUserMember.outputs.isExcluded }}" - - name: Add community and triage lables - if: steps.checkUserMember.outputs.isTeamMember != 'true' && steps.checkUserMember.outputs.isExcluded != 'true' - uses: AlexanderWert/issue-labeler@v2.3 + token: ${{ secrets.APM_TECH_USER_TOKEN }} + - name: Add community and triage labels + if: contains(steps.is_elastic_member.outputs.result, 'false') && github.actor != 'dependabot[bot]' && github.actor != 'apmmachine' + uses: actions-ecosystem/action-add-labels@v1 with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" - configuration-path: .github/community-label.yml - enable-versioned-regex: 0 - - name: Assign new internal pull requests to project - uses: elastic/assign-one-project-github-action@1.2.2 - if: (steps.checkUserMember.outputs.isTeamMember == 'true' || steps.checkUserMember.outputs.isExcluded == 'true') && github.event.pull_request - with: - project: 'https://github.com/orgs/elastic/projects/454' - project_id: '5882982' - column_name: 'In Progress' + labels: + - community + - triage From 6155e8a25f06efc7069799ec4bef10b097827f5b Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Thu, 14 Mar 2024 15:36:24 +0100 Subject: [PATCH 034/103] security: add permissions block to workflows (#1972) * security: add permissions block to workflows * Update .github/workflows/test-reporter.yml Co-authored-by: Victor Martinez * Update .github/workflows/labeler.yml * Remove permissions This will be removed in another PR --------- Co-authored-by: Victor Martinez --- .github/workflows/build-distribution.yml | 3 +++ .github/workflows/packages.yml | 3 +++ .github/workflows/pre-commit.yml | 3 +++ .github/workflows/run-matrix.yml | 3 +++ .github/workflows/test-reporter.yml | 5 +++++ .github/workflows/test.yml | 3 +++ 6 files changed, 20 insertions(+) diff --git a/.github/workflows/build-distribution.yml b/.github/workflows/build-distribution.yml index fd3e11ed7..ba8497e0c 100644 --- a/.github/workflows/build-distribution.yml +++ b/.github/workflows/build-distribution.yml @@ -3,6 +3,9 @@ name: build-distribution on: workflow_call: ~ +permissions: + contents: read + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index f2d9a4f83..3f63467fe 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -13,6 +13,9 @@ on: - '**/*.md' - '**/*.asciidoc' +permissions: + contents: read + # Override the version if there is no tag release. env: ELASTIC_CI_POST_VERSION: ${{ startsWith(github.ref, 'refs/tags') && '' || github.run_id }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 65947d33b..926c21be6 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -5,6 +5,9 @@ on: push: branches: [main] +permissions: + contents: read + jobs: pre-commit: runs-on: ubuntu-latest diff --git a/.github/workflows/run-matrix.yml b/.github/workflows/run-matrix.yml index 827212527..d5db311d6 100644 --- a/.github/workflows/run-matrix.yml +++ b/.github/workflows/run-matrix.yml @@ -8,6 +8,9 @@ on: description: Matrix include JSON string type: string +permissions: + contents: read + jobs: docker: name: "docker (version: ${{ matrix.version }}, framework: ${{ matrix.framework }})" diff --git a/.github/workflows/test-reporter.yml b/.github/workflows/test-reporter.yml index 4b0b7620d..1060771c5 100644 --- a/.github/workflows/test-reporter.yml +++ b/.github/workflows/test-reporter.yml @@ -8,6 +8,11 @@ on: types: - completed +permissions: + contents: read + actions: read + checks: write + jobs: report: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4638ab5d3..cae9a846e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,6 +30,9 @@ on: required: true type: boolean +permissions: + contents: read + jobs: build-distribution: uses: ./.github/workflows/build-distribution.yml From 26e934531c8b0d25b23f7bfc1ad79da57320b7db Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 14 Mar 2024 15:37:04 +0100 Subject: [PATCH 035/103] ci: remove adding issues to the old APM agents board (#2004) --- .github/workflows/addToProject.yml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .github/workflows/addToProject.yml diff --git a/.github/workflows/addToProject.yml b/.github/workflows/addToProject.yml deleted file mode 100644 index 0a3b76924..000000000 --- a/.github/workflows/addToProject.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Auto Assign to Project(s) - -on: - issues: - types: [opened, edited, milestoned] -env: - MY_GITHUB_TOKEN: ${{ secrets.APM_TECH_USER_TOKEN }} - -jobs: - assign_one_project: - runs-on: ubuntu-latest - name: Assign milestoned to Project - steps: - - name: Assign issues with milestones to project - uses: elastic/assign-one-project-github-action@1.2.2 - if: github.event.issue && github.event.issue.milestone - with: - project: 'https://github.com/orgs/elastic/projects/454' - project_id: '5882982' - column_name: 'Planned' From 495b04c741a92f0f169187aa02caea74df1c714c Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Mon, 18 Mar 2024 16:19:29 -0700 Subject: [PATCH 036/103] docs: Update flask.asciidoc (#2006) --- docs/flask.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/flask.asciidoc b/docs/flask.asciidoc index 9fb8f7e3e..b99ddd198 100644 --- a/docs/flask.asciidoc +++ b/docs/flask.asciidoc @@ -76,7 +76,7 @@ apm = ElasticAPM(app, service_name='', secret_token='') ===== Debug mode NOTE: Please note that errors and transactions will only be sent to the APM Server if your app is *not* in -http://flask.pocoo.org/docs/2.3.x/quickstart/#debug-mode[Flask debug mode]. +https://flask.palletsprojects.com/en/3.0.x/quickstart/#debug-mode[Flask debug mode]. To force the agent to send data while the app is in debug mode, set the value of `DEBUG` in the `ELASTIC_APM` dictionary to `True`: From 1872b8fef5a0f1a6f122c525f8ce8c568ec3bf31 Mon Sep 17 00:00:00 2001 From: Trent Mick Date: Tue, 19 Mar 2024 01:16:39 -0700 Subject: [PATCH 037/103] ci: fix labeler workflow syntax (#2007) Refs: #2003 --- .github/workflows/labeler.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 564cc7d31..a14b036c0 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -5,7 +5,7 @@ on: pull_request_target: types: [opened] -# 'issues: write' for https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#add-labels-to-an-issue +# '*: write' permissions for https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#add-labels-to-an-issue permissions: contents: read issues: write @@ -28,6 +28,6 @@ jobs: if: contains(steps.is_elastic_member.outputs.result, 'false') && github.actor != 'dependabot[bot]' && github.actor != 'apmmachine' uses: actions-ecosystem/action-add-labels@v1 with: - labels: - - community - - triage + labels: | + community + triage From 86eeee1e0fb51600c844c6d2dd89f11fd008f75a Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 19 Mar 2024 10:35:35 +0100 Subject: [PATCH 038/103] Add Django 4.2 LTS and Django 5.0 to test matrix (#1985) * ci: add Django 4.2 LTS and Django 5.0 to test matrix Closes #1946 * tests: remove django tests for old and not tested versions --- .ci/.matrix_exclude.yml | 22 ++++++ .ci/.matrix_framework.yml | 5 +- .ci/.matrix_framework_full.yml | 3 + tests/contrib/django/django_tests.py | 70 +------------------ tests/requirements/reqs-celery-5-django-5.txt | 2 + tests/requirements/reqs-django-4.2.txt | 3 + tests/requirements/reqs-django-5.0.txt | 3 + 7 files changed, 39 insertions(+), 69 deletions(-) create mode 100644 tests/requirements/reqs-celery-5-django-5.txt create mode 100644 tests/requirements/reqs-django-4.2.txt create mode 100644 tests/requirements/reqs-django-5.0.txt diff --git a/.ci/.matrix_exclude.yml b/.ci/.matrix_exclude.yml index 5eb71a09f..b3e735566 100644 --- a/.ci/.matrix_exclude.yml +++ b/.ci/.matrix_exclude.yml @@ -9,12 +9,34 @@ exclude: FRAMEWORK: django-4.0 - VERSION: python-3.7 FRAMEWORK: django-4.0 + # Django 4.2 requires Python 3.8+ + - VERSION: python-3.6 + FRAMEWORK: django-4.2 + - VERSION: python-3.7 + FRAMEWORK: django-4.2 + # Django 5.0 requires Python 3.10+ + - VERSION: python-3.6 + FRAMEWORK: django-5.0 + - VERSION: python-3.7 + FRAMEWORK: django-5.0 + - VERSION: python-3.8 + FRAMEWORK: django-5.0 + - VERSION: python-3.9 + FRAMEWORK: django-5.0 - VERSION: pypy-3 # current pypy-3 is compatible with Python 3.7 FRAMEWORK: celery-5-django-4 - VERSION: python-3.6 FRAMEWORK: celery-5-django-4 - VERSION: python-3.7 FRAMEWORK: celery-5-django-4 + - VERSION: python-3.6 + FRAMEWORK: celery-5-django-5 + - VERSION: python-3.7 + FRAMEWORK: celery-5-django-5 + - VERSION: python-3.8 + FRAMEWORK: celery-5-django-5 + - VERSION: python-3.9 + FRAMEWORK: celery-5-django-5 # Flask - VERSION: pypy-3 FRAMEWORK: flask-0.11 # see https://github.com/pallets/flask/commit/6e46d0cd, 0.11.2 was never released diff --git a/.ci/.matrix_framework.yml b/.ci/.matrix_framework.yml index 6bf64ab44..007827327 100644 --- a/.ci/.matrix_framework.yml +++ b/.ci/.matrix_framework.yml @@ -3,10 +3,10 @@ FRAMEWORK: - none - django-1.11 - - django-2.0 - - django-3.1 - django-3.2 - django-4.0 + - django-4.2 + - django-5.0 - flask-0.12 - flask-1.1 - flask-2.0 @@ -18,6 +18,7 @@ FRAMEWORK: - celery-4-django-2.0 - celery-5-flask-2 - celery-5-django-4 + - celery-5-django-5 - requests-newest - boto3-newest - pymongo-newest diff --git a/.ci/.matrix_framework_full.yml b/.ci/.matrix_framework_full.yml index 7b1ee213e..1cb11ae16 100644 --- a/.ci/.matrix_framework_full.yml +++ b/.ci/.matrix_framework_full.yml @@ -10,6 +10,8 @@ FRAMEWORK: - django-3.1 - django-3.2 - django-4.0 + - django-4.2 + - django-5.0 # - django-master - flask-0.10 - flask-0.11 @@ -25,6 +27,7 @@ FRAMEWORK: - celery-5-flask-2 - celery-5-django-3 - celery-5-django-4 + - celery-5-django-5 - opentelemetry-newest - opentracing-newest - opentracing-2.0 diff --git a/tests/contrib/django/django_tests.py b/tests/contrib/django/django_tests.py index 547d46b51..535729bcf 100644 --- a/tests/contrib/django/django_tests.py +++ b/tests/contrib/django/django_tests.py @@ -270,25 +270,7 @@ def test_user_info_with_custom_user_non_string_username(django_elasticapm_client assert user_info["username"] == "1" -@pytest.mark.skipif(django.VERSION > (1, 9), reason="MIDDLEWARE_CLASSES removed in Django 2.0") def test_user_info_with_non_django_auth(django_elasticapm_client, client): - with override_settings( - INSTALLED_APPS=[app for app in settings.INSTALLED_APPS if app != "django.contrib.auth"] - ) and override_settings( - MIDDLEWARE_CLASSES=[ - m for m in settings.MIDDLEWARE_CLASSES if m != "django.contrib.auth.middleware.AuthenticationMiddleware" - ] - ): - with pytest.raises(Exception): - resp = client.get(reverse("elasticapm-raise-exc")) - - assert len(django_elasticapm_client.events[ERROR]) == 1 - event = django_elasticapm_client.events[ERROR][0] - assert event["context"]["user"] == {} - - -@pytest.mark.skipif(django.VERSION < (1, 10), reason="MIDDLEWARE new in Django 1.10") -def test_user_info_with_non_django_auth_django_2(django_elasticapm_client, client): with override_settings( INSTALLED_APPS=[app for app in settings.INSTALLED_APPS if app != "django.contrib.auth"] ) and override_settings( @@ -303,22 +285,7 @@ def test_user_info_with_non_django_auth_django_2(django_elasticapm_client, clien assert event["context"]["user"] == {} -@pytest.mark.skipif(django.VERSION > (1, 9), reason="MIDDLEWARE_CLASSES removed in Django 2.0") def test_user_info_without_auth_middleware(django_elasticapm_client, client): - with override_settings( - MIDDLEWARE_CLASSES=[ - m for m in settings.MIDDLEWARE_CLASSES if m != "django.contrib.auth.middleware.AuthenticationMiddleware" - ] - ): - with pytest.raises(Exception): - client.get(reverse("elasticapm-raise-exc")) - assert len(django_elasticapm_client.events[ERROR]) == 1 - event = django_elasticapm_client.events[ERROR][0] - assert event["context"]["user"] == {} - - -@pytest.mark.skipif(django.VERSION < (1, 10), reason="MIDDLEWARE new in Django 1.10") -def test_user_info_without_auth_middleware_django_2(django_elasticapm_client, client): with override_settings( MIDDLEWARE_CLASSES=None, MIDDLEWARE=[m for m in settings.MIDDLEWARE if m != "django.contrib.auth.middleware.AuthenticationMiddleware"], @@ -614,8 +581,7 @@ def read(): assert_any_record_contains(caplog.records, "Can't capture request body: foobar") -@pytest.mark.skipif(django.VERSION < (1, 9), reason="get-raw-uri-not-available") -def test_disallowed_hosts_error_django_19(django_elasticapm_client): +def test_disallowed_hosts_error(django_elasticapm_client): request = WSGIRequest( environ={ "wsgi.input": io.BytesIO(), @@ -634,26 +600,6 @@ def test_disallowed_hosts_error_django_19(django_elasticapm_client): assert event["context"]["request"]["url"]["full"] == "http://testserver/" -@pytest.mark.skipif(django.VERSION >= (1, 9), reason="get-raw-uri-available") -def test_disallowed_hosts_error_django_18(django_elasticapm_client): - request = WSGIRequest( - environ={ - "wsgi.input": io.BytesIO(), - "wsgi.url_scheme": "http", - "REQUEST_METHOD": "POST", - "SERVER_NAME": "testserver", - "SERVER_PORT": "80", - "CONTENT_TYPE": "application/json", - "ACCEPT": "application/json", - } - ) - with override_settings(ALLOWED_HOSTS=["example.com"]): - # this should not raise a DisallowedHost exception - django_elasticapm_client.capture("Message", message="foo", request=request) - event = django_elasticapm_client.events[ERROR][0] - assert event["context"]["request"]["url"] == {"full": "DisallowedHost"} - - @pytest.mark.parametrize( "django_elasticapm_client", [{"capture_body": "errors"}, {"capture_body": "all"}, {"capture_body": "off"}], @@ -1196,16 +1142,6 @@ def test_stacktrace_filtered_for_elasticapm(client, django_elasticapm_client): assert spans[1]["stacktrace"][0]["module"].startswith("django.template"), spans[1]["stacktrace"][0]["function"] -@pytest.mark.skipif(django.VERSION > (1, 7), reason="argparse raises CommandError in this case") -@mock.patch("elasticapm.contrib.django.management.commands.elasticapm.Command._get_argv") -def test_subcommand_not_set(argv_mock): - stdout = io.StringIO() - argv_mock.return_value = ["manage.py", "elasticapm"] - call_command("elasticapm", stdout=stdout) - output = stdout.getvalue() - assert "No command specified" in output - - @mock.patch("elasticapm.contrib.django.management.commands.elasticapm.Command._get_argv") def test_subcommand_not_known(argv_mock): stdout = io.StringIO() @@ -1317,8 +1253,8 @@ def test_settings_server_url_with_credentials(): @pytest.mark.skipif( - not ((1, 10) <= django.VERSION < (2, 0)), - reason="only needed in 1.10 and 1.11 when both middleware settings are valid", + django.VERSION >= (2, 0), + reason="only needed in 1.11 when both middleware settings are valid", ) def test_django_1_10_uses_deprecated_MIDDLEWARE_CLASSES(): stdout = io.StringIO() diff --git a/tests/requirements/reqs-celery-5-django-5.txt b/tests/requirements/reqs-celery-5-django-5.txt new file mode 100644 index 000000000..b528dcb85 --- /dev/null +++ b/tests/requirements/reqs-celery-5-django-5.txt @@ -0,0 +1,2 @@ +-r reqs-celery-5.txt +-r reqs-django-5.0.txt diff --git a/tests/requirements/reqs-django-4.2.txt b/tests/requirements/reqs-django-4.2.txt new file mode 100644 index 000000000..6818ea895 --- /dev/null +++ b/tests/requirements/reqs-django-4.2.txt @@ -0,0 +1,3 @@ +Django>=4.2,<5.0 +jinja2<4 +-r reqs-base.txt diff --git a/tests/requirements/reqs-django-5.0.txt b/tests/requirements/reqs-django-5.0.txt new file mode 100644 index 000000000..dd2e1cea6 --- /dev/null +++ b/tests/requirements/reqs-django-5.0.txt @@ -0,0 +1,3 @@ +Django>=5.0,<5.1 +jinja2<4 +-r reqs-base.txt From 5289beb35129340f1cfe10952d3b8f6e0ca98939 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 19 Mar 2024 17:49:40 +0100 Subject: [PATCH 039/103] update CHANGELOG and bump version to 6.21.4 (#2008) --- CHANGELOG.asciidoc | 8 ++++++++ elasticapm/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 57a6cb95f..588030c14 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -32,6 +32,14 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.21.4]] +==== 6.21.4 - 2024-03-19 + +[float] +===== Bug fixes + +* Fix urllib3 2.0.1+ crash with many args {pull}2002[#2002] + [[release-notes-6.21.3]] ==== 6.21.3 - 2024-03-08 diff --git a/elasticapm/version.py b/elasticapm/version.py index 6da6c370d..d2ebe8c73 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 21, 3) +__version__ = (6, 21, 4) VERSION = ".".join(map(str, __version__)) From 43cdcfb4f9fd452764931b94d5e99522f12bf416 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 21 Mar 2024 09:56:22 +0100 Subject: [PATCH 040/103] docs: Add Django 4.2 and 5.0 to supported (#2010) --- docs/supported-technologies.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/supported-technologies.asciidoc b/docs/supported-technologies.asciidoc index 9a3b314d1..51358b0fd 100644 --- a/docs/supported-technologies.asciidoc +++ b/docs/supported-technologies.asciidoc @@ -42,6 +42,8 @@ We support these Django versions: * 3.1 * 3.2 * 4.0 + * 4.2 + * 5.0 For upcoming Django versions, we generally aim to ensure compatibility starting with the first Release Candidate. From 4f6696a04bc33f17c6726a8dc7df18dd89ba93ef Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 21 Mar 2024 09:56:41 +0100 Subject: [PATCH 041/103] ci: add Flask 2.1+ and 3.0 to matrix (#2011) --- .ci/.matrix_exclude.yml | 12 ++++++++++++ .ci/.matrix_framework.yml | 4 ++-- .ci/.matrix_framework_full.yml | 4 ++++ .gitignore | 1 + docs/supported-technologies.asciidoc | 4 ++++ tests/requirements/reqs-flask-2.0.txt | 2 +- tests/requirements/reqs-flask-2.1.txt | 4 ++++ tests/requirements/reqs-flask-2.2.txt | 4 ++++ tests/requirements/reqs-flask-2.3.txt | 4 ++++ tests/requirements/reqs-flask-3.0.txt | 3 +++ 10 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 tests/requirements/reqs-flask-2.1.txt create mode 100644 tests/requirements/reqs-flask-2.2.txt create mode 100644 tests/requirements/reqs-flask-2.3.txt create mode 100644 tests/requirements/reqs-flask-3.0.txt diff --git a/.ci/.matrix_exclude.yml b/.ci/.matrix_exclude.yml index b3e735566..eb7d57779 100644 --- a/.ci/.matrix_exclude.yml +++ b/.ci/.matrix_exclude.yml @@ -40,6 +40,18 @@ exclude: # Flask - VERSION: pypy-3 FRAMEWORK: flask-0.11 # see https://github.com/pallets/flask/commit/6e46d0cd, 0.11.2 was never released + - VERSION: python-3.6 + FRAMEWORK: flask-2.1 + - VERSION: python-3.6 + FRAMEWORK: flask-2.2 + - VERSION: python-3.6 + FRAMEWORK: flask-2.3 + - VERSION: python-3.6 + FRAMEWORK: flask-3.0 + - VERSION: python-3.7 + FRAMEWORK: flask-2.3 + - VERSION: python-3.7 + FRAMEWORK: flask-3.0 # Python 3.10 removed a bunch of classes from collections, now in collections.abc - VERSION: python-3.10 FRAMEWORK: django-1.11 diff --git a/.ci/.matrix_framework.yml b/.ci/.matrix_framework.yml index 007827327..df04f639c 100644 --- a/.ci/.matrix_framework.yml +++ b/.ci/.matrix_framework.yml @@ -8,8 +8,8 @@ FRAMEWORK: - django-4.2 - django-5.0 - flask-0.12 - - flask-1.1 - - flask-2.0 + - flask-2.3 + - flask-3.0 - jinja2-3 - opentelemetry-newest - opentracing-newest diff --git a/.ci/.matrix_framework_full.yml b/.ci/.matrix_framework_full.yml index 1cb11ae16..cb4361711 100644 --- a/.ci/.matrix_framework_full.yml +++ b/.ci/.matrix_framework_full.yml @@ -19,6 +19,10 @@ FRAMEWORK: - flask-1.0 - flask-1.1 - flask-2.0 + - flask-2.1 + - flask-2.2 + - flask-2.3 + - flask-3.0 - jinja2-2 - jinja2-3 - celery-4-flask-1.0 diff --git a/.gitignore b/.gitignore index 88e0a400b..65dc23f5e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.egg *.db *.pid +*.swp .coverage* .DS_Store .idea diff --git a/docs/supported-technologies.asciidoc b/docs/supported-technologies.asciidoc index 51358b0fd..f170617a2 100644 --- a/docs/supported-technologies.asciidoc +++ b/docs/supported-technologies.asciidoc @@ -61,6 +61,10 @@ We support these Flask versions: * 1.0 * 1.1 * 2.0 + * 2.1 + * 2.2 + * 2.3 + * 3.0 [float] [[supported-aiohttp]] diff --git a/tests/requirements/reqs-flask-2.0.txt b/tests/requirements/reqs-flask-2.0.txt index d68be1afa..62397f9b5 100644 --- a/tests/requirements/reqs-flask-2.0.txt +++ b/tests/requirements/reqs-flask-2.0.txt @@ -1,4 +1,4 @@ -Flask>=2.0,<3 +Flask>=2.0,<2.1 blinker>=1.1 itsdangerous -r reqs-base.txt diff --git a/tests/requirements/reqs-flask-2.1.txt b/tests/requirements/reqs-flask-2.1.txt new file mode 100644 index 000000000..84d89b8b9 --- /dev/null +++ b/tests/requirements/reqs-flask-2.1.txt @@ -0,0 +1,4 @@ +Flask>=2.1,<2.2 +blinker>=1.1 +itsdangerous +-r reqs-base.txt diff --git a/tests/requirements/reqs-flask-2.2.txt b/tests/requirements/reqs-flask-2.2.txt new file mode 100644 index 000000000..0b244a851 --- /dev/null +++ b/tests/requirements/reqs-flask-2.2.txt @@ -0,0 +1,4 @@ +Flask>=2.2,<2.3 +blinker>=1.1 +itsdangerous +-r reqs-base.txt diff --git a/tests/requirements/reqs-flask-2.3.txt b/tests/requirements/reqs-flask-2.3.txt new file mode 100644 index 000000000..08434994e --- /dev/null +++ b/tests/requirements/reqs-flask-2.3.txt @@ -0,0 +1,4 @@ +Flask>=2.3,<3 +blinker>=1.1 +itsdangerous +-r reqs-base.txt diff --git a/tests/requirements/reqs-flask-3.0.txt b/tests/requirements/reqs-flask-3.0.txt new file mode 100644 index 000000000..92120aa14 --- /dev/null +++ b/tests/requirements/reqs-flask-3.0.txt @@ -0,0 +1,3 @@ +Flask>=3.0,<3.1 +itsdangerous +-r reqs-base.txt From cfc6a24832754e8b839b7e1c7ce0a1f3efac700c Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 22 Mar 2024 19:37:56 +0100 Subject: [PATCH 042/103] CONTRIBUTING: add more hints on what is required to cut a release (#1989) * CONTRIBUTING: open PRs to the main branch * CONTRIBUTING: add more hints on what you need to setup for releasing Fixes #1991 --- CONTRIBUTING.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f31f6c3c9..014b4828d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ Once your changes are ready to submit for review: 1. Submit a pull request - Push your local changes to your forked copy of the repository and [submit a pull request](https://help.github.com/articles/using-pull-requests). + Push your local changes to your forked copy of the repository and [submit a pull request](https://help.github.com/articles/using-pull-requests) to the `main` branch. In the pull request, choose a title which sums up the changes that you have made, and in the body provide more details about what your changes do. @@ -174,6 +174,11 @@ should "Squash and merge". ### Releasing +Releases tags are signed so you need to have a PGP key set up, you can follow Github documentation on [creating a key](https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key) and +on [telling git about it](https://docs.github.com/en/authentication/managing-commit-signature-verification/telling-git-about-your-signing-key). Alternatively you can sign with a SSH key, remember you have to upload your key +again even if you want to use the same key you are using for authorization. +Then make sure you have SSO figured out for the key you are using to push to github, see [Github documentation](https://docs.github.com/articles/authenticating-to-a-github-organization-with-saml-single-sign-on/). + If you have commit access, the process is as follows: 1. Update the version in `elasticapm/version.py` according to the scale of the change. (major, minor or patch) @@ -182,13 +187,14 @@ If you have commit access, the process is as follows: 1. For Majors: Add the new major version to `conf.yaml` in the [elastic/docs](https://github.com/elastic/docs) repo. 1. Commit changes with message `update CHANGELOG and bump version to X.Y.Z` where `X.Y.Z` is the version in `elasticapm/version.py` -1. Open a PR against `main` with these changes +1. Open a PR against `main` with these changes leaving the body empty 1. Once the PR is merged, fetch and checkout `upstream/main` 1. Tag the commit with `git tag -s vX.Y.Z`, for example `git tag -s v1.2.3`. Copy the changelog for the release to the tag message, removing any leading `#`. -1. Reset the current major branch (`1.x`, `2.x` etc) to point to the current main, e.g. `git branch -f 1.x main` 1. Push tag upstream with `git push upstream --tags` (and optionally to your own fork as well) -1. Update major branch, e.g. `1.x` on upstream with `git push upstream 1.x` +1. Open a PR from `main` to the major branch, e.g. `1.x` to update it. In order to keep history you may want to + merge with the `rebase` strategy. It is crucial that `main` and the major branch have the same content. 1. After tests pass, Github Actions will automatically build and push the new release to PyPI. 1. Edit and publish the [draft Github release](https://github.com/elastic/apm-agent-python/releases) - created by Github Actions. Copy the changelog into the body of the release. + created by Github Actions. Substitute the generated changelog with one hand written into the body of the + release and move the agent layer ARNs under a `
` block with a `summary`. From 46a48f1f23370973b98cfb08c57f0af3ced4219a Mon Sep 17 00:00:00 2001 From: Adrien Mannocci Date: Wed, 27 Mar 2024 14:05:24 +0100 Subject: [PATCH 043/103] fix: only set ELASTIC_CI_POST_VERSION when needed (Closes #2015) (#2016) Signed-off-by: Adrien Mannocci --- .github/workflows/packages.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 3f63467fe..4f04d78a4 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -16,10 +16,6 @@ on: permissions: contents: read -# Override the version if there is no tag release. -env: - ELASTIC_CI_POST_VERSION: ${{ startsWith(github.ref, 'refs/tags') && '' || github.run_id }} - jobs: build: runs-on: ubuntu-latest @@ -28,6 +24,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.10" + - name: Override the version if there is no tag release. + run: | + if [[ "${GITHUB_REF}" != refs/tags/* ]]; then + echo "ELASTIC_CI_POST_VERSION=${{ github.run_id }}" >> "${GITHUB_ENV}" + fi - name: Install wheel run: pip install --user wheel - name: Building universal wheel @@ -41,4 +42,3 @@ jobs: path: | dist/*.whl dist/*tar.gz - From 52c3a1f75098777c4a68021e561539865e727096 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Mar 2024 12:05:14 +0100 Subject: [PATCH 044/103] build(deps): bump the github-actions group with 1 update (#2013) Bumps the github-actions group with 1 update: [geekyeggo/delete-artifact](https://github.com/geekyeggo/delete-artifact). Updates `geekyeggo/delete-artifact` from 4.1.0 to 5.0.0 - [Release notes](https://github.com/geekyeggo/delete-artifact/releases) - [Changelog](https://github.com/GeekyEggo/delete-artifact/blob/main/CHANGELOG.md) - [Commits](https://github.com/geekyeggo/delete-artifact/compare/65041433121f7239077fa20be14c0690f70569de...24928e75e6e6590170563b8ddae9fac674508aa1) --- updated-dependencies: - dependency-name: geekyeggo/delete-artifact dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cae9a846e..381c317b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -215,6 +215,6 @@ jobs: with: name: html-coverage-report path: htmlcov - - uses: geekyeggo/delete-artifact@65041433121f7239077fa20be14c0690f70569de + - uses: geekyeggo/delete-artifact@24928e75e6e6590170563b8ddae9fac674508aa1 with: name: coverage-reports From 6590cd253964f4c43d7e6cb3fb939bd8e7d7f617 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 29 Mar 2024 21:55:35 +0100 Subject: [PATCH 045/103] Add ability to override default serialization (#2018) * utils: add simplejson based JSON handling Add an alternative json serialization and deserialization implementation using simplejson configured to be JSON compliant by converting floats like nan, +inf and -inf to null . The idea is to permit users to use a different serialization if they need a different behaviour. Refs #1886 * client: fix overriding of transport json serializer We get a string from config so need to import it. --- elasticapm/base.py | 3 +- elasticapm/utils/json_encoder.py | 6 +- elasticapm/utils/simplejson_encoder.py | 58 +++++++++++++++ tests/client/client_tests.py | 13 ++++ tests/requirements/reqs-base.txt | 1 + tests/utils/json_utils/tests.py | 7 ++ tests/utils/json_utils/tests_simplejson.py | 86 ++++++++++++++++++++++ 7 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 elasticapm/utils/simplejson_encoder.py create mode 100644 tests/utils/json_utils/tests_simplejson.py diff --git a/elasticapm/base.py b/elasticapm/base.py index 5f50dc79d..ff2fd6e25 100644 --- a/elasticapm/base.py +++ b/elasticapm/base.py @@ -155,7 +155,8 @@ def __init__(self, config=None, **inline) -> None: "processors": self.load_processors(), } if config.transport_json_serializer: - transport_kwargs["json_serializer"] = config.transport_json_serializer + json_serializer_func = import_string(config.transport_json_serializer) + transport_kwargs["json_serializer"] = json_serializer_func self._api_endpoint_url = urllib.parse.urljoin( self.config.server_url if self.config.server_url.endswith("/") else self.config.server_url + "/", diff --git a/elasticapm/utils/json_encoder.py b/elasticapm/utils/json_encoder.py index c40e0accd..3918bb233 100644 --- a/elasticapm/utils/json_encoder.py +++ b/elasticapm/utils/json_encoder.py @@ -31,13 +31,9 @@ import datetime import decimal +import json import uuid -try: - import json -except ImportError: - import simplejson as json - class BetterJSONEncoder(json.JSONEncoder): ENCODERS = { diff --git a/elasticapm/utils/simplejson_encoder.py b/elasticapm/utils/simplejson_encoder.py new file mode 100644 index 000000000..f538ffdac --- /dev/null +++ b/elasticapm/utils/simplejson_encoder.py @@ -0,0 +1,58 @@ +# BSD 3-Clause License +# +# Copyright (c) 2012, the Sentry Team, see AUTHORS for more details +# Copyright (c) 2019, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + + +import simplejson as json + +from elasticapm.utils.json_encoder import BetterJSONEncoder + + +class BetterSimpleJSONEncoder(json.JSONEncoder): + ENCODERS = BetterJSONEncoder.ENCODERS + + def default(self, obj): + if type(obj) in self.ENCODERS: + return self.ENCODERS[type(obj)](obj) + try: + return super(BetterSimpleJSONEncoder, self).default(obj) + except TypeError: + return str(obj) + + +def better_decoder(data): + return data + + +def dumps(value, **kwargs): + return json.dumps(value, cls=BetterSimpleJSONEncoder, ignore_nan=True, **kwargs) + + +def loads(value, **kwargs): + return json.loads(value, object_hook=better_decoder) diff --git a/tests/client/client_tests.py b/tests/client/client_tests.py index af266b710..84e473fc8 100644 --- a/tests/client/client_tests.py +++ b/tests/client/client_tests.py @@ -48,6 +48,11 @@ import elasticapm from elasticapm.base import Client from elasticapm.conf.constants import ERROR + +try: + from elasticapm.utils.simplejson_encoder import dumps as simplejson_dumps +except ImportError: + simplejson_dumps = None from tests.fixtures import DummyTransport, TempStoreClient from tests.utils import assert_any_record_contains @@ -228,6 +233,14 @@ def test_custom_transport(elasticapm_client): assert isinstance(elasticapm_client._transport, DummyTransport) +@pytest.mark.skipIf(simplejson_dumps is None) +@pytest.mark.parametrize( + "elasticapm_client", [{"transport_json_serializer": "elasticapm.utils.simplejson_encoder.dumps"}], indirect=True +) +def test_custom_transport_json_serializer(elasticapm_client): + assert elasticapm_client._transport._json_serializer == simplejson_dumps + + @pytest.mark.parametrize("elasticapm_client", [{"processors": []}], indirect=True) def test_empty_processor_list(elasticapm_client): assert elasticapm_client.processors == [] diff --git a/tests/requirements/reqs-base.txt b/tests/requirements/reqs-base.txt index 42bac1bb8..747e42631 100644 --- a/tests/requirements/reqs-base.txt +++ b/tests/requirements/reqs-base.txt @@ -30,6 +30,7 @@ pytz ecs_logging structlog wrapt>=1.14.1,<1.15.0 +simplejson pytest-asyncio==0.21.0 ; python_version >= '3.7' asynctest==0.13.0 ; python_version >= '3.7' diff --git a/tests/utils/json_utils/tests.py b/tests/utils/json_utils/tests.py index 7cbef4b36..28791e79d 100644 --- a/tests/utils/json_utils/tests.py +++ b/tests/utils/json_utils/tests.py @@ -36,6 +36,8 @@ import decimal import uuid +import pytest + from elasticapm.utils import json_encoder as json @@ -69,6 +71,11 @@ def test_decimal(): assert json.dumps(res) == "1.0" +@pytest.mark.parametrize("res", [float("nan"), float("+inf"), float("-inf")]) +def test_float_invalid_json(res): + assert json.dumps(res) != "null" + + def test_unsupported(): res = object() assert json.dumps(res).startswith('" Date: Tue, 2 Apr 2024 10:00:41 +0200 Subject: [PATCH 046/103] tests/client: fix pytest mark skipif syntax (#2019) For some reason tests passed with this but now are failing because pytest test does not recognize skipIf. --- tests/client/client_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/client/client_tests.py b/tests/client/client_tests.py index 84e473fc8..a61248c85 100644 --- a/tests/client/client_tests.py +++ b/tests/client/client_tests.py @@ -233,7 +233,7 @@ def test_custom_transport(elasticapm_client): assert isinstance(elasticapm_client._transport, DummyTransport) -@pytest.mark.skipIf(simplejson_dumps is None) +@pytest.mark.skipif(simplejson_dumps is None, reason="no test without simplejson") @pytest.mark.parametrize( "elasticapm_client", [{"transport_json_serializer": "elasticapm.utils.simplejson_encoder.dumps"}], indirect=True ) From 0f4cc9d360aeb81e080fab903524375139466dfc Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 3 Apr 2024 09:20:37 +0200 Subject: [PATCH 047/103] tests: require Werkzeug < 3 for flask 2.0 tests (#2020) Otherwise it'll skip tests becasue flask import does not work. --- tests/requirements/reqs-flask-2.0.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements/reqs-flask-2.0.txt b/tests/requirements/reqs-flask-2.0.txt index 62397f9b5..dc3cb6572 100644 --- a/tests/requirements/reqs-flask-2.0.txt +++ b/tests/requirements/reqs-flask-2.0.txt @@ -1,4 +1,5 @@ Flask>=2.0,<2.1 +Werkzeug<3 blinker>=1.1 itsdangerous -r reqs-base.txt From c2861b4f25c241bbe368a249f5ab75177cc5dcfa Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 3 Apr 2024 09:20:59 +0200 Subject: [PATCH 048/103] Use docker compose subcommand instead of docker-compose (#2022) --- CONTRIBUTING.md | 4 ++-- tests/scripts/docker/run_tests.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 014b4828d..21180e9f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -108,7 +108,7 @@ that is a fixture which is defined #### Adding new instrumentations to the matrix build For tests that require external dependencies like databases, or for testing different versions of the same library, -we use a matrix build that leverages Docker and docker-compose. +we use a matrix build that leverages Docker. The setup requires a little bit of boilerplate to get started. In this example, we will create an instrumentation for the "foo" database, by instrumenting its Python driver, `foodriver`. @@ -153,7 +153,7 @@ In this example, we will create an instrumentation for the "foo" database, by in image: foobase:latest You'll also have to add a `DOCKER_DEPS` environment variable to `tests/scripts/envs/foo.sh` which tells the matrix - to spin up the given docker-compose service before running your tests. + to spin up the given Docker compose service before running your tests. You may also need to add things like hostname configuration here. DOCKER_DEPS="foo" diff --git a/tests/scripts/docker/run_tests.sh b/tests/scripts/docker/run_tests.sh index 9a89f93be..de518dc32 100755 --- a/tests/scripts/docker/run_tests.sh +++ b/tests/scripts/docker/run_tests.sh @@ -2,7 +2,7 @@ set -ex function cleanup { - PYTHON_VERSION=${1} docker-compose down -v + PYTHON_VERSION=${1} docker compose down -v if [[ $CODECOV_TOKEN ]]; then cd .. @@ -42,7 +42,7 @@ echo "Running tests for ${1}/${2}" if [[ -n $DOCKER_DEPS ]] then - PYTHON_VERSION=${1} docker-compose up -d ${DOCKER_DEPS} + PYTHON_VERSION=${1} docker compose up -d ${DOCKER_DEPS} fi # CASS_DRIVER_NO_EXTENSIONS is set so we don't build the Cassandra C-extensions, @@ -57,7 +57,7 @@ if ! ${CI}; then . fi -PYTHON_VERSION=${1} docker-compose run \ +PYTHON_VERSION=${1} docker compose run \ -e PYTHON_FULL_VERSION=${1} \ -e LOCAL_USER_ID=$LOCAL_USER_ID \ -e LOCAL_GROUP_ID=$LOCAL_GROUP_ID \ From cb3ea8104a5685187eb410298a0414808701b4be Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 3 Apr 2024 09:38:55 +0200 Subject: [PATCH 049/103] Remove vendored wrapt references (#2021) --- .flake8 | 1 - .gitignore | 1 - pyproject.toml | 1 - tests/scripts/license_headers_check.sh | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index f629a9d29..7be07fbeb 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,5 @@ [flake8] exclude= - elasticapm/utils/wrapt/**, build/**, src/**, tests/**, diff --git a/.gitignore b/.gitignore index 65dc23f5e..12eae962b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ pip-log.txt /docs/doctrees /example_project/*.db tests/.schemacache -elasticapm/utils/wrapt/_wrappers*.so coverage .tox .eggs diff --git a/pyproject.toml b/pyproject.toml index 167532517..019a7b666 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,6 @@ exclude = ''' | _build | build | dist - | elasticapm/utils/wrapt # The following are specific to Black, you probably don't want those. | blib2to3 diff --git a/tests/scripts/license_headers_check.sh b/tests/scripts/license_headers_check.sh index dc239df96..9ba9655e0 100755 --- a/tests/scripts/license_headers_check.sh +++ b/tests/scripts/license_headers_check.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash if [[ $# -eq 0 ]] then - FILES=$(find . -iname "*.py" -not -path "./elasticapm/utils/wrapt/*" -not -path "./dist/*" -not -path "./build/*" -not -path "./tests/utils/stacks/linenos.py") + FILES=$(find . -iname "*.py" -not -path "./dist/*" -not -path "./build/*" -not -path "./tests/utils/stacks/linenos.py") else FILES=$@ fi From 871abd2b4c8f128aeb7e45932ff4c225382b6479 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 3 Apr 2024 17:04:30 +0200 Subject: [PATCH 050/103] update CHANGELOG and bump version to 6.22.0 (#2023) --- CHANGELOG.asciidoc | 8 ++++++++ elasticapm/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 588030c14..03f4e048f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -32,6 +32,14 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.22.0]] +==== 6.22.0 - 2024-04-03 + +[float] +===== Features + +* Add ability to override default JSON serialization {pull}2018[#2018] + [[release-notes-6.21.4]] ==== 6.21.4 - 2024-03-19 diff --git a/elasticapm/version.py b/elasticapm/version.py index d2ebe8c73..c8799817e 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 21, 4) +__version__ = (6, 22, 0) VERSION = ".".join(map(str, __version__)) From cba0caf8af884a688e83008cdb1b25dc0214c31f Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 4 Apr 2024 17:58:57 +0200 Subject: [PATCH 051/103] CONTRIBUTING: describe how to create the pr to the vX.x branch (#2026) Create a PR from a branch already rebased so that the commit list in the github UI is clean. --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21180e9f1..c58fbb7c6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -192,7 +192,8 @@ If you have commit access, the process is as follows: 1. Tag the commit with `git tag -s vX.Y.Z`, for example `git tag -s v1.2.3`. Copy the changelog for the release to the tag message, removing any leading `#`. 1. Push tag upstream with `git push upstream --tags` (and optionally to your own fork as well) -1. Open a PR from `main` to the major branch, e.g. `1.x` to update it. In order to keep history you may want to +1. Open a PR from `main` to the major branch, e.g. `1.x` to update it. In order to keep history create a + branch from the `main` branch, rebase it on top of the major branch to drop duplicated commits and then merge with the `rebase` strategy. It is crucial that `main` and the major branch have the same content. 1. After tests pass, Github Actions will automatically build and push the new release to PyPI. 1. Edit and publish the [draft Github release](https://github.com/elastic/apm-agent-python/releases) From cbec0051ffca78ccd9ef2128e19e080300720b03 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 10 Apr 2024 10:30:38 -0400 Subject: [PATCH 052/103] chore: APM agent json server schema a76e999543efb3ba803c9a57dd13a4f6b... (#2028) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ... 3ffa7e1 Made with ❤️️ by updatecli Co-authored-by: apmmachine --- tests/upstream/json-specs/metadata.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/upstream/json-specs/metadata.json b/tests/upstream/json-specs/metadata.json index 7103bbeb5..1122ed68c 100644 --- a/tests/upstream/json-specs/metadata.json +++ b/tests/upstream/json-specs/metadata.json @@ -441,6 +441,14 @@ ], "maxLength": 1024 }, + "host_id": { + "description": "The OpenTelemetry semantic conventions compliant \"host.id\" attribute, if available.", + "type": [ + "null", + "string" + ], + "maxLength": 1024 + }, "hostname": { "description": "Deprecated: Use ConfiguredHostname and DetectedHostname instead. DeprecatedHostname is the host name of the system the service is running on. It does not distinguish between configured and detected hostname and therefore is deprecated and only used if no other hostname information is available.", "type": [ From 55934029daa0c0156105d62372a1f7aa69109a48 Mon Sep 17 00:00:00 2001 From: timostrunk Date: Thu, 18 Apr 2024 13:40:23 +0200 Subject: [PATCH 053/103] Relaxed wrapt-dependency to only blacklist 1.15.0 (#2005) --- setup.cfg | 2 +- tests/requirements/reqs-base.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index ce33450a6..2dca4283e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,7 @@ zip_safe = false install_requires = urllib3!=2.0.0,<3.0.0 certifi - wrapt>=1.14.1,<1.15.0 # https://github.com/elastic/apm-agent-python/issues/1894 + wrapt>=1.14.1,!=1.15.0 # https://github.com/elastic/apm-agent-python/issues/1894 ecs_logging test_suite=tests diff --git a/tests/requirements/reqs-base.txt b/tests/requirements/reqs-base.txt index 747e42631..849469a79 100644 --- a/tests/requirements/reqs-base.txt +++ b/tests/requirements/reqs-base.txt @@ -29,7 +29,7 @@ mock pytz ecs_logging structlog -wrapt>=1.14.1,<1.15.0 +wrapt>=1.14.1,!=1.15.0 simplejson pytest-asyncio==0.21.0 ; python_version >= '3.7' From bcc0b64cced8b3cd9eced5a3acd59cbc153e19bf Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 23 Apr 2024 18:18:58 +0200 Subject: [PATCH 054/103] github-action: enable provenance (#2014) --- .github/actions/build-distribution/action.yml | 21 +++++++ .github/actions/packages/action.yml | 31 ++++++++++ .github/dependabot.yml | 27 ++++++++ .github/workflows/build-distribution.yml | 23 ------- .github/workflows/packages.yml | 25 ++------ .github/workflows/release.yml | 62 +++++++++++++------ .github/workflows/test.yml | 6 +- dev-utils/make-packages.sh | 16 +++++ 8 files changed, 147 insertions(+), 64 deletions(-) create mode 100644 .github/actions/build-distribution/action.yml create mode 100644 .github/actions/packages/action.yml delete mode 100644 .github/workflows/build-distribution.yml create mode 100755 dev-utils/make-packages.sh diff --git a/.github/actions/build-distribution/action.yml b/.github/actions/build-distribution/action.yml new file mode 100644 index 000000000..bc0d55c29 --- /dev/null +++ b/.github/actions/build-distribution/action.yml @@ -0,0 +1,21 @@ +--- + +name: common build distribution tasks +description: Run the build distribution + +runs: + using: "composite" + steps: + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Build lambda layer zip + run: ./dev-utils/make-distribution.sh + shell: bash + + - uses: actions/upload-artifact@v4 + with: + name: build-distribution + path: ./build/ + if-no-files-found: error diff --git a/.github/actions/packages/action.yml b/.github/actions/packages/action.yml new file mode 100644 index 000000000..3ffda1d8c --- /dev/null +++ b/.github/actions/packages/action.yml @@ -0,0 +1,31 @@ +--- + +name: common package tasks +description: Run the packages + +runs: + using: "composite" + steps: + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Override the version if there is no tag release. + run: | + if [[ "${GITHUB_REF}" != refs/tags/* ]]; then + echo "ELASTIC_CI_POST_VERSION=${{ github.run_id }}" >> "${GITHUB_ENV}" + fi + shell: bash + - name: Build packages + run: ./dev-utils/make-packages.sh + shell: bash + - name: Upload Packages + uses: actions/upload-artifact@v4 + with: + name: packages + path: | + dist/*.whl + dist/*tar.gz + - name: generate build provenance + uses: github-early-access/generate-build-provenance@main + with: + subject-path: "${{ github.workspace }}/dist/*" \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index eb1cff95b..afb941790 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,3 +29,30 @@ updates: github-actions: patterns: - "*" + + # GitHub composite actions + - package-ecosystem: "github-actions" + directory: "/.github/actions/packages" + reviewers: + - "elastic/observablt-ci" + schedule: + interval: "weekly" + day: "sunday" + time: "22:00" + groups: + github-actions: + patterns: + - "*" + + - package-ecosystem: "github-actions" + directory: "/.github/actions/build-distribution" + reviewers: + - "elastic/observablt-ci" + schedule: + interval: "weekly" + day: "sunday" + time: "22:00" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/build-distribution.yml b/.github/workflows/build-distribution.yml deleted file mode 100644 index ba8497e0c..000000000 --- a/.github/workflows/build-distribution.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: build-distribution - -on: - workflow_call: ~ - -permissions: - contents: read - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Build lambda layer zip - run: ./dev-utils/make-distribution.sh - - uses: actions/upload-artifact@v3 - with: - name: build-distribution - path: ./build/ - if-no-files-found: error diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 4f04d78a4..7e11b4a6c 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -18,27 +18,10 @@ permissions: jobs: build: + permissions: + id-token: write + contents: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Override the version if there is no tag release. - run: | - if [[ "${GITHUB_REF}" != refs/tags/* ]]; then - echo "ELASTIC_CI_POST_VERSION=${{ github.run_id }}" >> "${GITHUB_ENV}" - fi - - name: Install wheel - run: pip install --user wheel - - name: Building universal wheel - run: python setup.py bdist_wheel - - name: Building source distribution - run: python setup.py sdist - - name: Upload Packages - uses: actions/upload-artifact@v4 - with: - name: packages - path: | - dist/*.whl - dist/*tar.gz + - uses: ./.github/actions/packages diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f91014b4..616cd9d4d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,13 @@ jobs: enabled: ${{ startsWith(github.ref, 'refs/tags') }} packages: - uses: ./.github/workflows/packages.yml + permissions: + id-token: write + contents: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/packages publish-pypi: needs: @@ -46,7 +52,17 @@ jobs: repository-url: https://test.pypi.org/legacy/ build-distribution: - uses: ./.github/workflows/build-distribution.yml + permissions: + id-token: write + contents: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-distribution + - name: generate build provenance + uses: github-early-access/generate-build-provenance@main + with: + subject-path: "${{ github.workspace }}/build/dist/elastic-apm-python-lambda-layer.zip" publish-lambda-layers: needs: @@ -63,7 +79,7 @@ jobs: secrets: | secret/observability-team/ci/service-account/apm-agent-python access_key_id | AWS_ACCESS_KEY_ID ; secret/observability-team/ci/service-account/apm-agent-python secret_access_key | AWS_SECRET_ACCESS_KEY - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-distribution path: ./build @@ -86,6 +102,9 @@ jobs: needs: - build-distribution runs-on: ubuntu-latest + permissions: + id-token: write + contents: write env: DOCKER_IMAGE_NAME: docker.elastic.co/observability/apm-agent-python steps: @@ -97,7 +116,7 @@ jobs: url: ${{ secrets.VAULT_ADDR }} roleId: ${{ secrets.VAULT_ROLE_ID }} secretId: ${{ secrets.VAULT_SECRET_ID }} - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-distribution path: ./build @@ -107,25 +126,30 @@ jobs: if [ "${{ startsWith(github.ref, 'refs/tags') }}" == "false" ] ; then # for testing purposes echo "tag=test" >> "${GITHUB_OUTPUT}" + echo "latest=test-latest" >> "${GITHUB_OUTPUT}" else # version without v prefix (e.g. 1.2.3) echo "tag=${GITHUB_REF_NAME/v/}" >> "${GITHUB_OUTPUT}" + echo "latest=latest" >> "${GITHUB_OUTPUT}" fi - - name: Docker build - run: >- - docker build - -t ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} - --build-arg AGENT_DIR=./build/dist/package/python - . - - name: Docker retag - run: >- - docker tag - ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} - ${{ env.DOCKER_IMAGE_NAME }}:latest - - name: Docker push - if: startsWith(github.ref, 'refs/tags') - run: |- - docker push --all-tags ${{ env.DOCKER_IMAGE_NAME }} + - name: Build and push image + id: push + uses: docker/build-push-action@v5.3.0 + with: + context: . + push: true + tags: | + ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} + ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.latest }} + build-args: | + AGENT_DIR=./build/dist/package/python + + - name: Attest image + uses: github-early-access/generate-build-provenance@main + with: + subject-name: "${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }}" + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: false github-draft: permissions: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 381c317b0..967378697 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,11 @@ permissions: jobs: build-distribution: - uses: ./.github/workflows/build-distribution.yml + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-distribution + create-matrix: runs-on: ubuntu-latest diff --git a/dev-utils/make-packages.sh b/dev-utils/make-packages.sh new file mode 100755 index 000000000..91b2a7bd1 --- /dev/null +++ b/dev-utils/make-packages.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Make a Python APM agent distribution +# + +echo "::group::Install wheel" +pip install --user wheel +echo "::endgroup::" + +echo "::group::Building universal wheel" +python setup.py bdist_wheel +echo "::endgroup::" + +echo "::group::Building source distribution" +python setup.py sdist +echo "::endgroup::" From d546006b0de98aaf0d936fafacbb1a188ba8ddd3 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 24 Apr 2024 10:49:33 +0200 Subject: [PATCH 055/103] github-action: use container image for provenance (#2030) --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 616cd9d4d..96b8e279a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -144,10 +144,10 @@ jobs: build-args: | AGENT_DIR=./build/dist/package/python - - name: Attest image + - name: generate build provenance (containers) uses: github-early-access/generate-build-provenance@main with: - subject-name: "${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }}" + subject-name: "${{ env.DOCKER_IMAGE_NAME }}" subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: false From 59188184c27c5df164c30d7b2c844d1cea394669 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 26 Apr 2024 13:57:52 +0200 Subject: [PATCH 056/103] github-action: add attestations scope (#2032) --- .github/actions/packages/action.yml | 4 ---- .github/workflows/packages.yml | 3 --- .github/workflows/release.yml | 7 +++++++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/packages/action.yml b/.github/actions/packages/action.yml index 3ffda1d8c..871f49c32 100644 --- a/.github/actions/packages/action.yml +++ b/.github/actions/packages/action.yml @@ -25,7 +25,3 @@ runs: path: | dist/*.whl dist/*tar.gz - - name: generate build provenance - uses: github-early-access/generate-build-provenance@main - with: - subject-path: "${{ github.workspace }}/dist/*" \ No newline at end of file diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 7e11b4a6c..496107508 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -18,9 +18,6 @@ permissions: jobs: build: - permissions: - id-token: write - contents: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 96b8e279a..31932eb2e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,12 +19,17 @@ jobs: packages: permissions: + attestations: write id-token: write contents: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: ./.github/actions/packages + - name: generate build provenance + uses: github-early-access/generate-build-provenance@main + with: + subject-path: "${{ github.workspace }}/dist/*" publish-pypi: needs: @@ -53,6 +58,7 @@ jobs: build-distribution: permissions: + attestations: write id-token: write contents: write runs-on: ubuntu-latest @@ -103,6 +109,7 @@ jobs: - build-distribution runs-on: ubuntu-latest permissions: + attestations: write id-token: write contents: write env: From 9ffc15f43e7752bd7a2b5844c35dda843513b635 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 7 May 2024 15:41:59 +0200 Subject: [PATCH 057/103] ci: use docker/metadata-action to gather tags/labels (#2033) --- .github/workflows/release.yml | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31932eb2e..841057ac2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -123,31 +123,29 @@ jobs: url: ${{ secrets.VAULT_ADDR }} roleId: ${{ secrets.VAULT_ROLE_ID }} secretId: ${{ secrets.VAULT_SECRET_ID }} + - uses: actions/download-artifact@v4 with: name: build-distribution path: ./build - - id: setup-docker - name: Set up docker variables - run: |- - if [ "${{ startsWith(github.ref, 'refs/tags') }}" == "false" ] ; then - # for testing purposes - echo "tag=test" >> "${GITHUB_OUTPUT}" - echo "latest=test-latest" >> "${GITHUB_OUTPUT}" - else - # version without v prefix (e.g. 1.2.3) - echo "tag=${GITHUB_REF_NAME/v/}" >> "${GITHUB_OUTPUT}" - echo "latest=latest" >> "${GITHUB_OUTPUT}" - fi + + - name: Extract metadata (tags, labels) + id: docker-meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: ${{ env.DOCKER_IMAGE_NAME }} + tags: | + type=raw,value=latest,prefix=test-,enable={{is_default_branch}} + type=semver,pattern={{version}} + - name: Build and push image id: push - uses: docker/build-push-action@v5.3.0 + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 with: context: . push: true - tags: | - ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.tag }} - ${{ env.DOCKER_IMAGE_NAME }}:${{ steps.setup-docker.outputs.latest }} + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} build-args: | AGENT_DIR=./build/dist/package/python @@ -156,7 +154,7 @@ jobs: with: subject-name: "${{ env.DOCKER_IMAGE_NAME }}" subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: false + push-to-registry: true github-draft: permissions: From 2fc86424a09eee7876e64523195b83f0660d15c6 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 8 May 2024 09:13:44 +0200 Subject: [PATCH 058/103] tests: pick mysql image tagged as 8.0 (#2035) As for some reason latest 8.4.0 is not working. --- tests/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 48b43cda2..62a05c83f 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -130,7 +130,7 @@ services: - pymssqldata:/var/opt/mssql mysql: - image: mysql + image: mysql:8.0 command: --default-authentication-plugin=mysql_native_password --log_error_verbosity=3 environment: - MYSQL_DATABASE=eapm_tests From 54dc9c45ec7f7efca30f21e8dc7d87a542c9d0dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 May 2024 17:23:40 +0200 Subject: [PATCH 059/103] build(deps): bump jinja2 from 3.0.3 to 3.1.4 in /tests/requirements (#2034) * build(deps): bump jinja2 from 3.0.3 to 3.1.4 in /tests/requirements Bumps [jinja2](https://github.com/pallets/jinja) from 3.0.3 to 3.1.4. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.0.3...3.1.4) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Update tests/requirements/reqs-jinja2-2.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Riccardo Magliocchetti --- tests/requirements/reqs-asgi-2.txt | 2 +- tests/requirements/reqs-base.txt | 2 +- tests/requirements/reqs-flask-1.0.txt | 2 +- tests/requirements/reqs-flask-1.1.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/requirements/reqs-asgi-2.txt b/tests/requirements/reqs-asgi-2.txt index eecc89d9a..97ff3022d 100644 --- a/tests/requirements/reqs-asgi-2.txt +++ b/tests/requirements/reqs-asgi-2.txt @@ -1,6 +1,6 @@ quart==0.6.13 MarkupSafe<2.1 -jinja2==3.0.3 +jinja2==3.1.4 async-asgi-testclient asgiref -r reqs-base.txt diff --git a/tests/requirements/reqs-base.txt b/tests/requirements/reqs-base.txt index 849469a79..4f79a5929 100644 --- a/tests/requirements/reqs-base.txt +++ b/tests/requirements/reqs-base.txt @@ -8,7 +8,7 @@ coverage[toml]==6.3 ; python_version == '3.7' coverage==7.3.1 ; python_version > '3.7' pytest-cov==4.0.0 ; python_version < '3.8' pytest-cov==4.1.0 ; python_version > '3.7' -jinja2==3.1.2 ; python_version == '3.7' +jinja2==3.1.4 ; python_version == '3.7' pytest-localserver==0.5.0 pytest-mock==3.6.1 ; python_version == '3.6' pytest-mock==3.10.0 ; python_version > '3.6' diff --git a/tests/requirements/reqs-flask-1.0.txt b/tests/requirements/reqs-flask-1.0.txt index 029a82cdf..b81a74b02 100644 --- a/tests/requirements/reqs-flask-1.0.txt +++ b/tests/requirements/reqs-flask-1.0.txt @@ -1,4 +1,4 @@ -jinja2<3.1.0 +jinja2<3.2.0 Werkzeug<2.1.0 Flask>=1.0,<1.1 MarkupSafe<2.1 diff --git a/tests/requirements/reqs-flask-1.1.txt b/tests/requirements/reqs-flask-1.1.txt index 107d375d5..cd32a4696 100644 --- a/tests/requirements/reqs-flask-1.1.txt +++ b/tests/requirements/reqs-flask-1.1.txt @@ -1,4 +1,4 @@ -jinja2<3.1.0 +jinja2<3.2.0 Werkzeug<2.1.0 Flask>=1.1,<1.2 MarkupSafe<2.1 From 4659c3607bca74bf32816083e178f3cf69c66ba6 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 10 May 2024 09:43:27 +0200 Subject: [PATCH 060/103] tests: revert bumping jinja2 in flask 1.0 tests (#2037) Unfortunately flask 1.0 does not work with jinja 3.1.x: venv/lib/python3.10/site-packages/flask/__init__.py:19: in from jinja2 import Markup, escape E ImportError: cannot import name 'Markup' from 'jinja2' --- tests/requirements/reqs-flask-1.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements/reqs-flask-1.0.txt b/tests/requirements/reqs-flask-1.0.txt index b81a74b02..029a82cdf 100644 --- a/tests/requirements/reqs-flask-1.0.txt +++ b/tests/requirements/reqs-flask-1.0.txt @@ -1,4 +1,4 @@ -jinja2<3.2.0 +jinja2<3.1.0 Werkzeug<2.1.0 Flask>=1.0,<1.1 MarkupSafe<2.1 From 285dbc5d669ac256133b0bdbe012bd6a5f5fb2e0 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 14 May 2024 09:31:05 +0200 Subject: [PATCH 061/103] Assume os.getppid is always available (#2038) It's always available since 3.2 and tests are already assuming that. --- elasticapm/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticapm/base.py b/elasticapm/base.py index ff2fd6e25..2c82f0d88 100644 --- a/elasticapm/base.py +++ b/elasticapm/base.py @@ -374,7 +374,7 @@ def get_service_info(self): def get_process_info(self): result = { "pid": os.getpid(), - "ppid": os.getppid() if hasattr(os, "getppid") else None, + "ppid": os.getppid(), "title": None, # Note: if we implement this, the value needs to be wrapped with keyword_field } if self.config.include_process_args: From 76a8e09638e677dafa8f24aa8fef1d4a2e8b9b93 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 14 May 2024 17:47:44 +0200 Subject: [PATCH 062/103] github-action: delete opentelemetry workflow (#2039) --- .github/workflows/opentelemetry.yml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .github/workflows/opentelemetry.yml diff --git a/.github/workflows/opentelemetry.yml b/.github/workflows/opentelemetry.yml deleted file mode 100644 index 84a6209ff..000000000 --- a/.github/workflows/opentelemetry.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Look up results at https://ela.st/oblt-ci-cd-stats. -# There will be one service per GitHub repository, including the org name, and one Transaction per Workflow. -name: OpenTelemetry Export Trace - -on: - workflow_run: - workflows: [ "*" ] - types: [completed] - -permissions: - contents: read - -jobs: - otel-export-trace: - runs-on: ubuntu-latest - steps: - - uses: elastic/apm-pipeline-library/.github/actions/opentelemetry@current - with: - vaultUrl: ${{ secrets.VAULT_ADDR }} - vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} - vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} From 9aaa6c968febd991c993238e14714d068fb076b9 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 14 May 2024 21:32:12 +0200 Subject: [PATCH 063/103] ci: build and push Docker image based on Chainguard base image (#2036) --- .github/workflows/release.yml | 7 +++++++ Dockerfile.wolfi | 3 +++ 2 files changed, 10 insertions(+) create mode 100644 Dockerfile.wolfi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 841057ac2..854ee4231 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -112,6 +112,10 @@ jobs: attestations: write id-token: write contents: write + strategy: + fail-fast: false + matrix: + dockerfile: [ 'Dockerfile', 'Dockerfile.wolfi' ] env: DOCKER_IMAGE_NAME: docker.elastic.co/observability/apm-agent-python steps: @@ -137,6 +141,8 @@ jobs: tags: | type=raw,value=latest,prefix=test-,enable={{is_default_branch}} type=semver,pattern={{version}} + flavor: | + suffix=${{ contains(matrix.dockerfile, 'wolfi') && '-wolfi' || '' }} - name: Build and push image id: push @@ -144,6 +150,7 @@ jobs: with: context: . push: true + file: ${{ matrix.dockerfile }} tags: ${{ steps.docker-meta.outputs.tags }} labels: ${{ steps.docker-meta.outputs.labels }} build-args: | diff --git a/Dockerfile.wolfi b/Dockerfile.wolfi new file mode 100644 index 000000000..1ed923ce5 --- /dev/null +++ b/Dockerfile.wolfi @@ -0,0 +1,3 @@ +FROM docker.elastic.co/wolfi/chainguard-base@sha256:9f940409f96296ef56140bcc4665c204dd499af4c32c96cc00e792558097c3f1 +ARG AGENT_DIR +COPY ${AGENT_DIR} /opt/python \ No newline at end of file From 5adcc5e16296925c24e2ef2631d328d4180d56da Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 15 May 2024 09:04:42 +0200 Subject: [PATCH 064/103] github-action: use actions/attest-build-provenance (#2040) --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 854ee4231..cc0de40a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/packages - name: generate build provenance - uses: github-early-access/generate-build-provenance@main + uses: actions/attest-build-provenance@951c0c5f8e375ad4efad33405ab77f7ded2358e4 # v1.1.1 with: subject-path: "${{ github.workspace }}/dist/*" @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/build-distribution - name: generate build provenance - uses: github-early-access/generate-build-provenance@main + uses: actions/attest-build-provenance@951c0c5f8e375ad4efad33405ab77f7ded2358e4 # v1.1.1 with: subject-path: "${{ github.workspace }}/build/dist/elastic-apm-python-lambda-layer.zip" @@ -157,7 +157,7 @@ jobs: AGENT_DIR=./build/dist/package/python - name: generate build provenance (containers) - uses: github-early-access/generate-build-provenance@main + uses: actions/attest-build-provenance@951c0c5f8e375ad4efad33405ab77f7ded2358e4 # v1.1.1 with: subject-name: "${{ env.DOCKER_IMAGE_NAME }}" subject-digest: ${{ steps.push.outputs.digest }} From 07cf90bed0cff24a4d8134f102ebcab4712aa81f Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 16 May 2024 10:57:45 +0200 Subject: [PATCH 065/103] contrib/serverless: remove aws reference from azure implementation (#2042) --- elasticapm/contrib/serverless/azure.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/elasticapm/contrib/serverless/azure.py b/elasticapm/contrib/serverless/azure.py index ed2444d60..c5df4882a 100644 --- a/elasticapm/contrib/serverless/azure.py +++ b/elasticapm/contrib/serverless/azure.py @@ -43,8 +43,6 @@ from elasticapm.utils.disttracing import TraceParent from elasticapm.utils.logging import get_logger -SERVERLESS_HTTP_REQUEST = ("api", "elb") - logger = get_logger("elasticapm.serverless") _AnnotatedFunctionT = TypeVar("_AnnotatedFunctionT") From 62a0a7ea342ccb5bea6694084cca6a1118b009ef Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 17 May 2024 15:40:27 +0200 Subject: [PATCH 066/103] update CHANGELOG and bump version to 6.22.1 (#2044) --- CHANGELOG.asciidoc | 8 ++++++++ elasticapm/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 03f4e048f..cf6a99a7c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -32,6 +32,14 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.22.1]] +==== 6.22.1 - 2024-05-17 + +[float] +===== Features + +* Relax wrapt dependency to only exclude 1.15.0 {pull}2005[#2005] + [[release-notes-6.22.0]] ==== 6.22.0 - 2024-04-03 diff --git a/elasticapm/version.py b/elasticapm/version.py index c8799817e..e778fc5b2 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 22, 0) +__version__ = (6, 22, 1) VERSION = ".".join(map(str, __version__)) From 6fee7253ca929f635b13c0006448a56fd227d6cc Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 17 May 2024 16:48:54 +0200 Subject: [PATCH 067/103] ci: use v3 artifact APIs for build-distribution (#2046) Hopefully fix: The lambda layer can be published as follows for dev work: aws lambda --output json publish-layer-version --layer-name 'runner-dev-elastic-apm-python' --description 'runner dev Elastic APM Python agent lambda layer' --zip-file 'fileb://build/dist/elastic-apm-python-lambda-layer.zip' Run actions/upload-artifact@v4 with: name: build-distribution path: ./build/ if-no-files-found: error compression-level: 6 overwrite: false env: ... With the provided path, there will be 636 files uploaded Artifact name is valid! Root directory input is valid! Error: Failed to CreateArtifact: Received non-retryable error: Failed request: (409) Conflict: an artifact with this name already exists on the workflow run --- .github/actions/build-distribution/action.yml | 2 +- .github/workflows/release.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-distribution/action.yml b/.github/actions/build-distribution/action.yml index bc0d55c29..05c32eeb8 100644 --- a/.github/actions/build-distribution/action.yml +++ b/.github/actions/build-distribution/action.yml @@ -14,7 +14,7 @@ runs: run: ./dev-utils/make-distribution.sh shell: bash - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v3 with: name: build-distribution path: ./build/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cc0de40a2..c00975d7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -85,7 +85,7 @@ jobs: secrets: | secret/observability-team/ci/service-account/apm-agent-python access_key_id | AWS_ACCESS_KEY_ID ; secret/observability-team/ci/service-account/apm-agent-python secret_access_key | AWS_SECRET_ACCESS_KEY - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v3 with: name: build-distribution path: ./build @@ -128,7 +128,7 @@ jobs: roleId: ${{ secrets.VAULT_ROLE_ID }} secretId: ${{ secrets.VAULT_SECRET_ID }} - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v3 with: name: build-distribution path: ./build From f466707713fb17de80b135e401993cacab217e6a Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Mon, 20 May 2024 10:25:02 +0200 Subject: [PATCH 068/103] update CHANGELOG and bump version to 6.22.2 (#2048) --- CHANGELOG.asciidoc | 8 ++++++++ elasticapm/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index cf6a99a7c..45934551a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -32,6 +32,14 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.22.2]] +==== 6.22.2 - 2024-05-20 + +[float] +===== Bug fixes + +* Fix CI release workflow {pull}2046[#2046] + [[release-notes-6.22.1]] ==== 6.22.1 - 2024-05-17 diff --git a/elasticapm/version.py b/elasticapm/version.py index e778fc5b2..9d26934f4 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 22, 1) +__version__ = (6, 22, 2) VERSION = ".".join(map(str, __version__)) From 96907dd7aa242f782ed2b348d596234a2f7c5f5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 11:11:40 +0200 Subject: [PATCH 069/103] --- (#2050) updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tests/requirements/reqs-starlette-0.13.txt | 2 +- tests/requirements/reqs-starlette-0.14.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/requirements/reqs-starlette-0.13.txt b/tests/requirements/reqs-starlette-0.13.txt index 3144bf464..51a5cd486 100644 --- a/tests/requirements/reqs-starlette-0.13.txt +++ b/tests/requirements/reqs-starlette-0.13.txt @@ -1,4 +1,4 @@ starlette>=0.13,<0.14 aiofiles==0.7.0 -requests==2.31.0 +requests==2.32.0 -r reqs-base.txt diff --git a/tests/requirements/reqs-starlette-0.14.txt b/tests/requirements/reqs-starlette-0.14.txt index e1952d09b..a075a72f3 100644 --- a/tests/requirements/reqs-starlette-0.14.txt +++ b/tests/requirements/reqs-starlette-0.14.txt @@ -1,4 +1,4 @@ starlette>=0.14,<0.15 -requests==2.31.0 +requests==2.32.0 aiofiles -r reqs-base.txt From 377d3692b88d4812d6c9c7499ff32e79bba2b6bb Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 22 May 2024 16:24:01 +0200 Subject: [PATCH 070/103] Fix starlette requirements (#2051) * tests: fix ancient starlette requirements Newer requests dropped support for Pyrhon < 3.8 so keep an old version for these requirements. Hopefully it'll also be useful as reminder next time dependabot will bump a release. * ci: run starlette-0.14 tests before merging To avoid regressions with dependabot bumps --- .ci/.matrix_framework.yml | 2 ++ tests/requirements/reqs-starlette-0.13.txt | 3 ++- tests/requirements/reqs-starlette-0.14.txt | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.ci/.matrix_framework.yml b/.ci/.matrix_framework.yml index df04f639c..6eff578d8 100644 --- a/.ci/.matrix_framework.yml +++ b/.ci/.matrix_framework.yml @@ -45,6 +45,8 @@ FRAMEWORK: - aiopg-newest - asyncpg-newest - tornado-newest + # this has a dependency on requests, run it to catch update issues before merging. Drop after baseline > 0.21.0 + - starlette-0.14 - starlette-newest - pymemcache-newest - graphene-2 diff --git a/tests/requirements/reqs-starlette-0.13.txt b/tests/requirements/reqs-starlette-0.13.txt index 51a5cd486..43d814c60 100644 --- a/tests/requirements/reqs-starlette-0.13.txt +++ b/tests/requirements/reqs-starlette-0.13.txt @@ -1,4 +1,5 @@ starlette>=0.13,<0.14 aiofiles==0.7.0 -requests==2.32.0 +requests==2.32.1; python_version >= '3.8' +requests==2.31.0; python_version < '3.8' -r reqs-base.txt diff --git a/tests/requirements/reqs-starlette-0.14.txt b/tests/requirements/reqs-starlette-0.14.txt index a075a72f3..52ea93114 100644 --- a/tests/requirements/reqs-starlette-0.14.txt +++ b/tests/requirements/reqs-starlette-0.14.txt @@ -1,4 +1,5 @@ starlette>=0.14,<0.15 -requests==2.32.0 +requests==2.32.1; python_version >= '3.8' +requests==2.31.0; python_version < '3.8' aiofiles -r reqs-base.txt From f26bdfe0b21e5638ce6b10e283548962c266cf67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 11:37:59 +0200 Subject: [PATCH 071/103] build(deps): bump actions/attest-build-provenance (#2052) Bumps the github-actions group with 1 update: [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance). Updates `actions/attest-build-provenance` from 1.1.1 to 1.1.2 - [Release notes](https://github.com/actions/attest-build-provenance/releases) - [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest-build-provenance/compare/951c0c5f8e375ad4efad33405ab77f7ded2358e4...173725a1209d09b31f9d30a3890cf2757ebbff0d) --- updated-dependencies: - dependency-name: actions/attest-build-provenance dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c00975d7c..b4fff9d53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/packages - name: generate build provenance - uses: actions/attest-build-provenance@951c0c5f8e375ad4efad33405ab77f7ded2358e4 # v1.1.1 + uses: actions/attest-build-provenance@173725a1209d09b31f9d30a3890cf2757ebbff0d # v1.1.2 with: subject-path: "${{ github.workspace }}/dist/*" @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/build-distribution - name: generate build provenance - uses: actions/attest-build-provenance@951c0c5f8e375ad4efad33405ab77f7ded2358e4 # v1.1.1 + uses: actions/attest-build-provenance@173725a1209d09b31f9d30a3890cf2757ebbff0d # v1.1.2 with: subject-path: "${{ github.workspace }}/build/dist/elastic-apm-python-lambda-layer.zip" @@ -157,7 +157,7 @@ jobs: AGENT_DIR=./build/dist/package/python - name: generate build provenance (containers) - uses: actions/attest-build-provenance@951c0c5f8e375ad4efad33405ab77f7ded2358e4 # v1.1.1 + uses: actions/attest-build-provenance@173725a1209d09b31f9d30a3890cf2757ebbff0d # v1.1.2 with: subject-name: "${{ env.DOCKER_IMAGE_NAME }}" subject-digest: ${{ steps.push.outputs.digest }} From 3df0cdddee922477813011e52c3a7499dd2b8c53 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 31 May 2024 10:40:39 +0200 Subject: [PATCH 072/103] ci: use updatecli with GitHub secrets (#2053) --- .../updatecli.d/update-gherkin-specs.yml | 10 ++++----- .../updatecli.d/update-json-specs.yml | 10 ++++----- .ci/updatecli/updatecli.d/update-specs.yml | 10 ++++----- .github/workflows/updatecli.yml | 21 +++++++++---------- 4 files changed, 22 insertions(+), 29 deletions(-) diff --git a/.ci/updatecli/updatecli.d/update-gherkin-specs.yml b/.ci/updatecli/updatecli.d/update-gherkin-specs.yml index f12ece861..82f986fae 100644 --- a/.ci/updatecli/updatecli.d/update-gherkin-specs.yml +++ b/.ci/updatecli/updatecli.d/update-gherkin-specs.yml @@ -5,22 +5,20 @@ scms: default: kind: github spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' owner: "{{ .github.owner }}" repository: "{{ .github.repository }}" token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' branch: "{{ .github.branch }}" apm: kind: github spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' owner: "{{ .github.owner }}" repository: "{{ .github.apm_repository }}" token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' branch: "{{ .github.branch }}" sources: diff --git a/.ci/updatecli/updatecli.d/update-json-specs.yml b/.ci/updatecli/updatecli.d/update-json-specs.yml index e05aaecdb..7b86367a7 100644 --- a/.ci/updatecli/updatecli.d/update-json-specs.yml +++ b/.ci/updatecli/updatecli.d/update-json-specs.yml @@ -5,22 +5,20 @@ scms: default: kind: github spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' owner: "{{ .github.owner }}" repository: "{{ .github.repository }}" token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' branch: "{{ .github.branch }}" apm: kind: github spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' owner: "{{ .github.owner }}" repository: "{{ .github.apm_repository }}" token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' branch: "{{ .github.branch }}" sources: diff --git a/.ci/updatecli/updatecli.d/update-specs.yml b/.ci/updatecli/updatecli.d/update-specs.yml index 554140da2..ab5589f7e 100644 --- a/.ci/updatecli/updatecli.d/update-specs.yml +++ b/.ci/updatecli/updatecli.d/update-specs.yml @@ -5,23 +5,21 @@ scms: default: kind: github spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' owner: "{{ .github.owner }}" repository: "{{ .github.repository }}" token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' branch: "{{ .github.branch }}" apm-data: kind: github spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' owner: "{{ .github.owner }}" repository: "{{ .github.apm_data_repository }}" token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' branch: "{{ .github.branch }}" sources: diff --git a/.github/workflows/updatecli.yml b/.github/workflows/updatecli.yml index 4fc00bd71..8190487ad 100644 --- a/.github/workflows/updatecli.yml +++ b/.github/workflows/updatecli.yml @@ -13,17 +13,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current + + - uses: elastic/oblt-actions/updatecli/run@v1 with: - vaultUrl: ${{ secrets.VAULT_ADDR }} - vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} - vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - pipeline: .ci/updatecli/updatecli.d - values: .ci/updatecli/values.yml + command: "apply --config .ci/updatecli/updatecli.d --values .ci/updatecli/values.yml" + env: + GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }} + - if: failure() - uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current + uses: elastic/oblt-actions/slack/send@v1 with: - vaultUrl: ${{ secrets.VAULT_ADDR }} - vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} - vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - slackChannel: "#apm-agent-python" + bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + channel-id: "#apm-agent-python" + message: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, @robots-ci please look what's going on " From 32aac6297474329d06ee894094b5501b777d7686 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 09:33:07 +0200 Subject: [PATCH 073/103] build(deps): bump certifi from 2024.2.2 to 2024.6.2 in /dev-utils (#2055) Bumps [certifi](https://github.com/certifi/python-certifi) from 2024.2.2 to 2024.6.2. - [Commits](https://github.com/certifi/python-certifi/compare/2024.02.02...2024.06.02) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev-utils/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-utils/requirements.txt b/dev-utils/requirements.txt index 59008afc2..acb4cdcaa 100644 --- a/dev-utils/requirements.txt +++ b/dev-utils/requirements.txt @@ -1,4 +1,4 @@ # These are the pinned requirements for the lambda layer/docker image -certifi==2024.2.2 +certifi==2024.6.2 urllib3==1.26.18 wrapt==1.14.1 From d8fce5c48a80b1dd9172de7f3e2a313843b48120 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 3 Jun 2024 10:21:18 +0200 Subject: [PATCH 074/103] github: use docker github secrets (#2054) --- .github/workflows/release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4fff9d53..2523499f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,13 +120,13 @@ jobs: DOCKER_IMAGE_NAME: docker.elastic.co/observability/apm-agent-python steps: - uses: actions/checkout@v4 - - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current + + - name: Log in to the Elastic Container registry + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 with: - registry: docker.elastic.co - secret: secret/observability-team/ci/docker-registry/prod - url: ${{ secrets.VAULT_ADDR }} - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} + registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }} + username: ${{ secrets.ELASTIC_DOCKER_USERNAME }} + password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }} - uses: actions/download-artifact@v3 with: From 26ce296400e3d2626a9ed30908fc71763d69a52a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 4 Jun 2024 18:39:17 +0200 Subject: [PATCH 075/103] github-actions: remove snapshoty (#2059) --- .ci/snapshoty.yml | 36 --------------------------------- .github/workflows/README.md | 2 +- .github/workflows/snapshoty.yml | 35 -------------------------------- 3 files changed, 1 insertion(+), 72 deletions(-) delete mode 100644 .ci/snapshoty.yml delete mode 100644 .github/workflows/snapshoty.yml diff --git a/.ci/snapshoty.yml b/.ci/snapshoty.yml deleted file mode 100644 index ccebc3426..000000000 --- a/.ci/snapshoty.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- - -# Version of configuration to use -version: '1.0' - -# You can define a Google Cloud Account to use -account: - # Project id of the service account - project: '${GCS_PROJECT}' - # Private key id of the service account - private_key_id: '${GCS_PRIVATE_KEY_ID}' - # Private key of the service account - private_key: '${GCS_PRIVATE_KEY}' - # Email of the service account - client_email: '${GCS_CLIENT_EMAIL}' - # URI token - token_uri: 'https://oauth2.googleapis.com/token' - -# List of artifacts -artifacts: - # Path to use for artifacts discovery - - path: './dist' - # Files pattern to match - files_pattern: 'elastic_apm-(?P\d+\.\d+\.\d+)-(.*)\.whl' - # File layout on GCS bucket - output_pattern: '{project}/{github_branch_name}/elastic-apm-python-{app_version}-{github_sha_short}.whl' - # List of metadata processors to use. - metadata: - # Define static custom metadata - - name: 'custom' - data: - project: 'apm-agent-python' - # Add git metadata - - name: 'git' - # Add github_actions metadata - - name: 'github_actions' diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c224d62b8..5e2641541 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -52,4 +52,4 @@ The tag release follows the naming convention: `v...`, wher ### OpenTelemetry -There is a GitHub workflow in charge to populate what the workflow run in terms of jobs and steps. Those details can be seen in [here](https://ela.st/oblt-ci-cd-stats) (**NOTE**: only available for Elasticians). +Every workflow and its logs are exported to OpenTelemetry traces/logs/metrics. Those details can be seen [here](https://ela.st/oblt-ci-cd-stats) (**NOTE**: only available for Elasticians). diff --git a/.github/workflows/snapshoty.yml b/.github/workflows/snapshoty.yml deleted file mode 100644 index 49d1b3423..000000000 --- a/.github/workflows/snapshoty.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -# Publish a snapshot. A "snapshot" is a packaging of the latest *unreleased* APM agent, -# published to a known GCS bucket for use in edge demo/test environments. -name: snapshoty - -on: - workflow_run: - workflows: - - test - types: - - completed - branches: - - main - -jobs: - packages: - if: ${{ github.event.workflow_run.conclusion == 'success' }} - uses: ./.github/workflows/packages.yml - upload: - needs: - - packages - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: packages - path: dist - - name: Publish snaphosts - uses: elastic/apm-pipeline-library/.github/actions/snapshoty-simple@current - with: - config: '.ci/snapshoty.yml' - vaultUrl: ${{ secrets.VAULT_ADDR }} - vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} - vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} From 6cca82cf0671195ff285d3b36555490c70a56d39 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 5 Jun 2024 09:19:59 +0200 Subject: [PATCH 076/103] github-action: run buildkite action with GH secrets (#2057) --- .github/workflows/microbenchmark.yml | 30 +++++++--------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/.github/workflows/microbenchmark.yml b/.github/workflows/microbenchmark.yml index 9af88a6b1..2230d7e41 100644 --- a/.github/workflows/microbenchmark.yml +++ b/.github/workflows/microbenchmark.yml @@ -16,32 +16,16 @@ permissions: jobs: microbenchmark: runs-on: ubuntu-latest - # wait up to 1 hour - timeout-minutes: 60 + timeout-minutes: 5 steps: - - id: buildkite - name: Run buildkite pipeline - uses: elastic/apm-pipeline-library/.github/actions/buildkite@current + - name: Run microbenchmark + uses: elastic/oblt-actions/buildkite/run@v1.5.0 with: - vaultUrl: ${{ secrets.VAULT_ADDR }} - vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} - vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - pipeline: apm-agent-microbenchmark - waitFor: true - printBuildLogs: true - buildEnvVars: | + pipeline: "apm-agent-microbenchmark" + token: ${{ secrets.BUILDKITE_TOKEN }} + wait-for: false + env-vars: | script=.ci/bench.sh repo=apm-agent-python sha=${{ github.sha }} BRANCH_NAME=${{ github.ref_name }} - - - if: ${{ failure() }} - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current - with: - url: ${{ secrets.VAULT_ADDR }} - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - channel: "#apm-agent-python" - message: | - :ghost: [${{ github.repository }}] microbenchmark *${{ github.ref_name }}* failed to run in Buildkite. - Build: (<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>) From d899ae0013fab487bb7f41defebe828e9e36b13d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 5 Jun 2024 17:53:03 +0200 Subject: [PATCH 077/103] feat(slack-bot): use github secrets (#2058) --- .github/workflows/release.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2523499f8..9b5f43faf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -198,11 +198,9 @@ jobs: uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current with: needs: ${{ toJSON(needs) }} - - uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current - if: startsWith(github.ref, 'refs/tags') + - if: startsWith(github.ref, 'refs/tags') + uses: elastic/oblt-actions/slack/notify-result@v1.7.0 with: + bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + channel-id: "#apm-agent-python" status: ${{ steps.check.outputs.status }} - vaultUrl: ${{ secrets.VAULT_ADDR }} - vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} - vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - slackChannel: "#apm-agent-python" From b5401a720a4465c99ce0579cc71d8d9c9fed27df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:01:29 +0200 Subject: [PATCH 078/103] build(deps): bump the github-actions group with 2 updates (#2061) Bumps the github-actions group with 2 updates: [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance) and [docker/login-action](https://github.com/docker/login-action). Updates `actions/attest-build-provenance` from 1.1.2 to 1.2.0 - [Release notes](https://github.com/actions/attest-build-provenance/releases) - [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest-build-provenance/compare/173725a1209d09b31f9d30a3890cf2757ebbff0d...49df96e17e918a15956db358890b08e61c704919) Updates `docker/login-action` from 3.1.0 to 3.2.0 - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/e92390c5fb421da1463c202d546fed0ec5c39f20...0d4c9c5ea7693da7b068278f7b52bda2a190a446) --- updated-dependencies: - dependency-name: actions/attest-build-provenance dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b5f43faf..d75b1627c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/packages - name: generate build provenance - uses: actions/attest-build-provenance@173725a1209d09b31f9d30a3890cf2757ebbff0d # v1.1.2 + uses: actions/attest-build-provenance@49df96e17e918a15956db358890b08e61c704919 # v1.2.0 with: subject-path: "${{ github.workspace }}/dist/*" @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/build-distribution - name: generate build provenance - uses: actions/attest-build-provenance@173725a1209d09b31f9d30a3890cf2757ebbff0d # v1.1.2 + uses: actions/attest-build-provenance@49df96e17e918a15956db358890b08e61c704919 # v1.2.0 with: subject-path: "${{ github.workspace }}/build/dist/elastic-apm-python-lambda-layer.zip" @@ -122,7 +122,7 @@ jobs: - uses: actions/checkout@v4 - name: Log in to the Elastic Container registry - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 with: registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }} username: ${{ secrets.ELASTIC_DOCKER_USERNAME }} @@ -157,7 +157,7 @@ jobs: AGENT_DIR=./build/dist/package/python - name: generate build provenance (containers) - uses: actions/attest-build-provenance@173725a1209d09b31f9d30a3890cf2757ebbff0d # v1.1.2 + uses: actions/attest-build-provenance@49df96e17e918a15956db358890b08e61c704919 # v1.2.0 with: subject-name: "${{ env.DOCKER_IMAGE_NAME }}" subject-digest: ${{ steps.push.outputs.digest }} From 0427172c11890b1fdea1f8f9b00b2f936253b8ec Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Mon, 10 Jun 2024 09:02:35 +0200 Subject: [PATCH 079/103] contrib/starlette: fix outcome for 500 responses without exception (#2060) * contrib/starlette: fix outcome for 500 responses without exception Set the outcome based on the status code of the http.response.start ASGI message instead of relying only on the raise of an exception. * contrib/asgi: fix outcome for 500 responses without exception Set the outcome based on the status code of the http.response.start ASGI message instead of relying only on the raise of an exception. --- elasticapm/contrib/asgi.py | 1 + elasticapm/contrib/starlette/__init__.py | 1 + tests/contrib/asgi/app.py | 5 +++++ tests/contrib/asgi/asgi_tests.py | 17 ++++++++++++++++ tests/contrib/asyncio/starlette_tests.py | 25 ++++++++++++++++++++++++ 5 files changed, 49 insertions(+) diff --git a/elasticapm/contrib/asgi.py b/elasticapm/contrib/asgi.py index 096fed36a..92ee6c193 100644 --- a/elasticapm/contrib/asgi.py +++ b/elasticapm/contrib/asgi.py @@ -50,6 +50,7 @@ async def wrapped_send(message) -> None: await set_context(lambda: middleware.get_data_from_response(message, constants.TRANSACTION), "response") result = "HTTP {}xx".format(message["status"] // 100) elasticapm.set_transaction_result(result, override=False) + elasticapm.set_transaction_outcome(http_status_code=message["status"], override=False) await send(message) return wrapped_send diff --git a/elasticapm/contrib/starlette/__init__.py b/elasticapm/contrib/starlette/__init__.py index ad26d7a0a..3dfb225c9 100644 --- a/elasticapm/contrib/starlette/__init__.py +++ b/elasticapm/contrib/starlette/__init__.py @@ -147,6 +147,7 @@ async def wrapped_send(message) -> None: ) result = "HTTP {}xx".format(message["status"] // 100) elasticapm.set_transaction_result(result, override=False) + elasticapm.set_transaction_outcome(http_status_code=message["status"], override=False) await send(message) _mocked_receive = None diff --git a/tests/contrib/asgi/app.py b/tests/contrib/asgi/app.py index a919b2cef..352720135 100644 --- a/tests/contrib/asgi/app.py +++ b/tests/contrib/asgi/app.py @@ -59,3 +59,8 @@ async def boom() -> None: @app.route("/body") async def json(): return jsonify({"hello": "world"}) + + +@app.route("/500", methods=["GET"]) +async def error(): + return "KO", 500 diff --git a/tests/contrib/asgi/asgi_tests.py b/tests/contrib/asgi/asgi_tests.py index 824a23b68..f2a096dcc 100644 --- a/tests/contrib/asgi/asgi_tests.py +++ b/tests/contrib/asgi/asgi_tests.py @@ -66,6 +66,23 @@ async def test_transaction_span(instrumented_app, elasticapm_client): assert span["sync"] == False +@pytest.mark.asyncio +async def test_transaction_span(instrumented_app, elasticapm_client): + async with async_asgi_testclient.TestClient(instrumented_app) as client: + resp = await client.get("/500") + assert resp.status_code == 500 + assert resp.text == "KO" + + assert len(elasticapm_client.events[constants.TRANSACTION]) == 1 + assert len(elasticapm_client.events[constants.SPAN]) == 0 + transaction = elasticapm_client.events[constants.TRANSACTION][0] + assert transaction["name"] == "GET unknown route" + assert transaction["result"] == "HTTP 5xx" + assert transaction["outcome"] == "failure" + assert transaction["context"]["request"]["url"]["full"] == "/500" + assert transaction["context"]["response"]["status_code"] == 500 + + @pytest.mark.asyncio async def test_transaction_ignore_url(instrumented_app, elasticapm_client): elasticapm_client.config.update("1", transaction_ignore_urls="/foo*") diff --git a/tests/contrib/asyncio/starlette_tests.py b/tests/contrib/asyncio/starlette_tests.py index fcd7d0dee..e3c4f4a16 100644 --- a/tests/contrib/asyncio/starlette_tests.py +++ b/tests/contrib/asyncio/starlette_tests.py @@ -110,6 +110,10 @@ async def with_slash(request): async def without_slash(request): return PlainTextResponse("Hi {}".format(request.path_params["name"])) + @app.route("/500/", methods=["GET"]) + async def with_500_status_code(request): + return PlainTextResponse("Oops", status_code=500) + @sub.route("/hi") async def hi_from_sub(request): return PlainTextResponse("sub") @@ -236,6 +240,27 @@ def test_exception(app, elasticapm_client): assert error["context"]["request"] == transaction["context"]["request"] +def test_failure_outcome_with_500_status_code(app, elasticapm_client): + client = TestClient(app) + + client.get("/500/") + + assert len(elasticapm_client.events[constants.TRANSACTION]) == 1 + transaction = elasticapm_client.events[constants.TRANSACTION][0] + spans = elasticapm_client.spans_for_transaction(transaction) + assert len(spans) == 0 + + assert transaction["name"] == "GET /500/" + assert transaction["result"] == "HTTP 5xx" + assert transaction["outcome"] == "failure" + assert transaction["type"] == "request" + request = transaction["context"]["request"] + assert request["method"] == "GET" + assert transaction["context"]["response"]["status_code"] == 500 + + assert len(elasticapm_client.events[constants.ERROR]) == 0 + + @pytest.mark.parametrize("header_name", [constants.TRACEPARENT_HEADER_NAME, constants.TRACEPARENT_LEGACY_HEADER_NAME]) def test_traceparent_handling(app, elasticapm_client, header_name): client = TestClient(app) From 25d9f6ff62c4cbebfce1e7153b7338a84dc46bad Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Mon, 10 Jun 2024 10:24:30 +0200 Subject: [PATCH 080/103] update CHANGELOG and bump version to 6.22.3 (#2062) --- CHANGELOG.asciidoc | 8 ++++++++ elasticapm/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 45934551a..b8df53ab6 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -32,6 +32,14 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.22.3]] +==== 6.22.3 - 2024-06-10 + +[float] +===== Bug fixes + +* Fix outcome in ASGI and Starlette apps on error status codes without an exception {pull}2060[#2060] + [[release-notes-6.22.2]] ==== 6.22.2 - 2024-05-20 diff --git a/elasticapm/version.py b/elasticapm/version.py index 9d26934f4..82aa446ad 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 22, 2) +__version__ = (6, 22, 3) VERSION = ".".join(map(str, __version__)) From a862591df6651752eb3793497b5b8197141e8a09 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jun 2024 10:26:41 +0200 Subject: [PATCH 081/103] github-actions: use slack and aws gh secrets (#2065) --- .github/workflows/release.yml | 13 ++++--------- .github/workflows/test.yml | 8 +++----- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d75b1627c..ea3dd63d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,17 +74,12 @@ jobs: needs: - build-distribution runs-on: ubuntu-latest + env: + # TODO: use keyless + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - uses: actions/checkout@v4 - - uses: hashicorp/vault-action@v3.0.0 - with: - url: ${{ secrets.VAULT_ADDR }} - method: approle - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - secrets: | - secret/observability-team/ci/service-account/apm-agent-python access_key_id | AWS_ACCESS_KEY_ID ; - secret/observability-team/ci/service-account/apm-agent-python secret_access_key | AWS_SECRET_ACCESS_KEY - uses: actions/download-artifact@v3 with: name: build-distribution diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 967378697..391d67f67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -174,13 +174,11 @@ jobs: needs: ${{ toJSON(needs) }} - run: ${{ steps.check.outputs.isSuccess }} - if: failure() && (github.event_name == 'schedule' || github.event_name == 'push') - uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current + uses: elastic/oblt-actions/slack/notify-result@v1 with: + bot-token: ${{ secrets.SLACK_BOT_TOKEN }} status: ${{ steps.check.outputs.status }} - vaultUrl: ${{ secrets.VAULT_ADDR }} - vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} - vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - slackChannel: "#apm-agent-python" + channel-id: "#apm-agent-python" coverage: name: Combine & check coverage. From cf21db01ed138f24ff0db43727ae5bfe75d17ed8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:04:32 +0200 Subject: [PATCH 082/103] build(deps): bump the github-actions group with 3 updates (#2066) Bumps the github-actions group with 3 updates: [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance), [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) and [docker/build-push-action](https://github.com/docker/build-push-action). Updates `actions/attest-build-provenance` from 1.2.0 to 1.3.1 - [Release notes](https://github.com/actions/attest-build-provenance/releases) - [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest-build-provenance/compare/49df96e17e918a15956db358890b08e61c704919...534b352d658f90498fd148d231fdbf88f3886a3a) Updates `pypa/gh-action-pypi-publish` from 1.8.14 to 1.9.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/81e9d935c883d0b210363ab89cf05f3894778450...ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0) Updates `docker/build-push-action` from 5.3.0 to 5.4.0 - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/2cdde995de11925a030ce8070c3d77a52ffcf1c0...ca052bb54ab0790a636c9b5f226502c73d547a25) --- updated-dependencies: - dependency-name: actions/attest-build-provenance dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea3dd63d9..b8fbe1686 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/packages - name: generate build provenance - uses: actions/attest-build-provenance@49df96e17e918a15956db358890b08e61c704919 # v1.2.0 + uses: actions/attest-build-provenance@534b352d658f90498fd148d231fdbf88f3886a3a # v1.3.1 with: subject-path: "${{ github.workspace }}/dist/*" @@ -47,12 +47,12 @@ jobs: path: dist - name: Upload pypi.org if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 + uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 with: repository-url: https://upload.pypi.org/legacy/ - name: Upload test.pypi.org if: ${{ ! startsWith(github.ref, 'refs/tags') }} - uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 + uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0 with: repository-url: https://test.pypi.org/legacy/ @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/build-distribution - name: generate build provenance - uses: actions/attest-build-provenance@49df96e17e918a15956db358890b08e61c704919 # v1.2.0 + uses: actions/attest-build-provenance@534b352d658f90498fd148d231fdbf88f3886a3a # v1.3.1 with: subject-path: "${{ github.workspace }}/build/dist/elastic-apm-python-lambda-layer.zip" @@ -141,7 +141,7 @@ jobs: - name: Build and push image id: push - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 with: context: . push: true @@ -152,7 +152,7 @@ jobs: AGENT_DIR=./build/dist/package/python - name: generate build provenance (containers) - uses: actions/attest-build-provenance@49df96e17e918a15956db358890b08e61c704919 # v1.2.0 + uses: actions/attest-build-provenance@534b352d658f90498fd148d231fdbf88f3886a3a # v1.3.1 with: subject-name: "${{ env.DOCKER_IMAGE_NAME }}" subject-digest: ${{ steps.push.outputs.digest }} From 239059c11b87d1919930e21ee488e3606332d391 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 09:06:24 +0200 Subject: [PATCH 083/103] build(deps): bump urllib3 from 1.26.18 to 1.26.19 in /dev-utils (#2067) Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.18 to 1.26.19. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.19/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.18...1.26.19) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev-utils/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-utils/requirements.txt b/dev-utils/requirements.txt index acb4cdcaa..87ab0cf28 100644 --- a/dev-utils/requirements.txt +++ b/dev-utils/requirements.txt @@ -1,4 +1,4 @@ # These are the pinned requirements for the lambda layer/docker image certifi==2024.6.2 -urllib3==1.26.18 +urllib3==1.26.19 wrapt==1.14.1 From bb4921e0aef5c9217384112251eccfd21361a321 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Mon, 24 Jun 2024 10:25:23 +0200 Subject: [PATCH 084/103] ci: remove celery 4 testing (#2068) With latest pip 24.1 Celery 4 is not installable anymore because the latest released version in 2020 contains invalid metadata: WARNING: Ignoring version 4.4.7 of celery since it has invalid metadata: Requested celery<5,>4.0 from celery-4.4.7-py2.py3-none-any.whl (from -r tests/requirements/reqs-celery-4.txt (line 1)) has invalid metadata: Expected matching RIGHT_PARENTHESIS for LEFT_PARENTHESIS, after version specifier pytz (>dev) ~^ Please use pip<24.1 if you need to use this version. --- .ci/.matrix_exclude.yml | 18 ------------------ .ci/.matrix_framework.yml | 2 -- .ci/.matrix_framework_full.yml | 3 --- docs/supported-technologies.asciidoc | 4 ++-- .../requirements/reqs-celery-4-django-1.11.txt | 2 -- .../requirements/reqs-celery-4-django-2.0.txt | 2 -- tests/requirements/reqs-celery-4-flask-1.0.txt | 2 -- tests/requirements/reqs-celery-4.txt | 4 ---- 8 files changed, 2 insertions(+), 35 deletions(-) delete mode 100644 tests/requirements/reqs-celery-4-django-1.11.txt delete mode 100644 tests/requirements/reqs-celery-4-django-2.0.txt delete mode 100644 tests/requirements/reqs-celery-4-flask-1.0.txt delete mode 100644 tests/requirements/reqs-celery-4.txt diff --git a/.ci/.matrix_exclude.yml b/.ci/.matrix_exclude.yml index eb7d57779..8df06914d 100644 --- a/.ci/.matrix_exclude.yml +++ b/.ci/.matrix_exclude.yml @@ -57,20 +57,12 @@ exclude: FRAMEWORK: django-1.11 - VERSION: python-3.10 FRAMEWORK: django-2.0 - - VERSION: python-3.10 - FRAMEWORK: celery-4-django-2.0 - - VERSION: python-3.10 - FRAMEWORK: celery-4-django-1.11 - - VERSION: python-3.11 # cannot import name 'formatargspec' from 'inspect' - FRAMEWORK: celery-4-flask-1.0 - VERSION: python-3.11 # https://github.com/celery/billiard/issues/377 FRAMEWORK: celery-5-flask-2 - VERSION: python-3.11 # https://github.com/celery/billiard/issues/377 FRAMEWORK: celery-5-django-3 - VERSION: python-3.11 # https://github.com/celery/billiard/issues/377 FRAMEWORK: celery-5-django-4 - - VERSION: python-3.12 # cannot import name 'formatargspec' from 'inspect' - FRAMEWORK: celery-4-flask-1.0 - VERSION: python-3.12 # https://github.com/celery/billiard/issues/377 FRAMEWORK: celery-5-flask-2 - VERSION: python-3.12 # https://github.com/celery/billiard/issues/377 @@ -93,10 +85,6 @@ exclude: FRAMEWORK: django-2.0 - VERSION: python-3.11 FRAMEWORK: django-2.1 - - VERSION: python-3.11 - FRAMEWORK: celery-4-django-2.0 - - VERSION: python-3.11 - FRAMEWORK: celery-4-django-1.11 - VERSION: python-3.11 FRAMEWORK: graphene-2 - VERSION: python-3.11 @@ -113,10 +101,6 @@ exclude: FRAMEWORK: django-2.0 - VERSION: python-3.12 FRAMEWORK: django-2.1 - - VERSION: python-3.12 - FRAMEWORK: celery-4-django-2.0 - - VERSION: python-3.12 - FRAMEWORK: celery-4-django-1.11 - VERSION: python-3.12 FRAMEWORK: graphene-2 - VERSION: python-3.12 @@ -294,8 +278,6 @@ exclude: FRAMEWORK: flask-1.1 - VERSION: python-3.7 FRAMEWORK: jinja2-2 - - VERSION: python-3.7 - FRAMEWORK: celery-4-flask-1.0 # TODO py3.12 - VERSION: python-3.12 FRAMEWORK: sanic-20.12 # no wheels available yet diff --git a/.ci/.matrix_framework.yml b/.ci/.matrix_framework.yml index 6eff578d8..679064a72 100644 --- a/.ci/.matrix_framework.yml +++ b/.ci/.matrix_framework.yml @@ -14,8 +14,6 @@ FRAMEWORK: - opentelemetry-newest - opentracing-newest - twisted-newest - - celery-4-flask-1.0 - - celery-4-django-2.0 - celery-5-flask-2 - celery-5-django-4 - celery-5-django-5 diff --git a/.ci/.matrix_framework_full.yml b/.ci/.matrix_framework_full.yml index cb4361711..d2482d9ff 100644 --- a/.ci/.matrix_framework_full.yml +++ b/.ci/.matrix_framework_full.yml @@ -25,9 +25,6 @@ FRAMEWORK: - flask-3.0 - jinja2-2 - jinja2-3 - - celery-4-flask-1.0 - - celery-4-django-1.11 - - celery-4-django-2.0 - celery-5-flask-2 - celery-5-django-3 - celery-5-django-4 diff --git a/docs/supported-technologies.asciidoc b/docs/supported-technologies.asciidoc index f170617a2..50198a102 100644 --- a/docs/supported-technologies.asciidoc +++ b/docs/supported-technologies.asciidoc @@ -128,8 +128,8 @@ The Python APM agent comes with automatic instrumentation of various 3rd party m We support these Celery versions: -* 3.x -* 4.x +* 4.x (deprecated) +* 5.x Celery tasks will be recorded automatically with Django and Flask only. diff --git a/tests/requirements/reqs-celery-4-django-1.11.txt b/tests/requirements/reqs-celery-4-django-1.11.txt deleted file mode 100644 index 4440bb70f..000000000 --- a/tests/requirements/reqs-celery-4-django-1.11.txt +++ /dev/null @@ -1,2 +0,0 @@ --r reqs-celery-4.txt --r reqs-django-1.11.txt diff --git a/tests/requirements/reqs-celery-4-django-2.0.txt b/tests/requirements/reqs-celery-4-django-2.0.txt deleted file mode 100644 index 72e805f38..000000000 --- a/tests/requirements/reqs-celery-4-django-2.0.txt +++ /dev/null @@ -1,2 +0,0 @@ --r reqs-celery-4.txt --r reqs-django-2.0.txt diff --git a/tests/requirements/reqs-celery-4-flask-1.0.txt b/tests/requirements/reqs-celery-4-flask-1.0.txt deleted file mode 100644 index e357a036f..000000000 --- a/tests/requirements/reqs-celery-4-flask-1.0.txt +++ /dev/null @@ -1,2 +0,0 @@ --r reqs-celery-4.txt --r reqs-flask-1.0.txt diff --git a/tests/requirements/reqs-celery-4.txt b/tests/requirements/reqs-celery-4.txt deleted file mode 100644 index 57ba4c638..000000000 --- a/tests/requirements/reqs-celery-4.txt +++ /dev/null @@ -1,4 +0,0 @@ -celery>4.0,<5 -# including future as it was missing in celery 4.4.4, see https://github.com/celery/celery/issues/6145 -future>=0.18.0 -importlib-metadata<5.0; python_version<"3.8" From 7606c2a40ff06b43c8b5bf8652bf8636849fc515 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 15:04:29 +0200 Subject: [PATCH 085/103] build(deps): bump the github-actions group with 2 updates (#2069) Bumps the github-actions group with 2 updates: [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance) and [docker/build-push-action](https://github.com/docker/build-push-action). Updates `actions/attest-build-provenance` from 1.3.1 to 1.3.2 - [Release notes](https://github.com/actions/attest-build-provenance/releases) - [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest-build-provenance/compare/534b352d658f90498fd148d231fdbf88f3886a3a...bdd51370e0416ac948727f861e03c2f05d32d78e) Updates `docker/build-push-action` from 5.4.0 to 6.1.0 - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/ca052bb54ab0790a636c9b5f226502c73d547a25...31159d49c0d4756269a0940a750801a1ea5d7003) --- updated-dependencies: - dependency-name: actions/attest-build-provenance dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b8fbe1686..36c909f4e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/packages - name: generate build provenance - uses: actions/attest-build-provenance@534b352d658f90498fd148d231fdbf88f3886a3a # v1.3.1 + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 with: subject-path: "${{ github.workspace }}/dist/*" @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/build-distribution - name: generate build provenance - uses: actions/attest-build-provenance@534b352d658f90498fd148d231fdbf88f3886a3a # v1.3.1 + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 with: subject-path: "${{ github.workspace }}/build/dist/elastic-apm-python-lambda-layer.zip" @@ -141,7 +141,7 @@ jobs: - name: Build and push image id: push - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + uses: docker/build-push-action@31159d49c0d4756269a0940a750801a1ea5d7003 # v6.1.0 with: context: . push: true @@ -152,7 +152,7 @@ jobs: AGENT_DIR=./build/dist/package/python - name: generate build provenance (containers) - uses: actions/attest-build-provenance@534b352d658f90498fd148d231fdbf88f3886a3a # v1.3.1 + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 with: subject-name: "${{ env.DOCKER_IMAGE_NAME }}" subject-digest: ${{ steps.push.outputs.digest }} From 3f8363d9e577d0029ed7608960df4b8e738b184d Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Tue, 25 Jun 2024 15:40:55 +0200 Subject: [PATCH 086/103] Use elastic/oblt-actions/aws/auth action to authenticate with AWS (#2070) --- .github/workflows/release.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 36c909f4e..d1fd242f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,16 +74,15 @@ jobs: needs: - build-distribution runs-on: ubuntu-latest - env: - # TODO: use keyless - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v3 with: name: build-distribution path: ./build + - uses: elastic/oblt-actions/aws/auth@v1 + with: + aws-account-id: "267093732750" - name: Publish lambda layers to AWS if: startsWith(github.ref, 'refs/tags') run: | From 9a6e79886ac4fa35eccdf659873fba6b2e936374 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Tue, 25 Jun 2024 16:43:41 +0200 Subject: [PATCH 087/103] Automate collapsing ARN table in release notes (#2071) * Automate collapsing ARN table in release notes * Apply suggestions from code review --------- Co-authored-by: Riccardo Magliocchetti --- .ci/create-arn-table.sh | 9 +++++++-- CONTRIBUTING.md | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.ci/create-arn-table.sh b/.ci/create-arn-table.sh index 3105822ea..a03ead4c6 100755 --- a/.ci/create-arn-table.sh +++ b/.ci/create-arn-table.sh @@ -10,7 +10,8 @@ AWS_FOLDER=${AWS_FOLDER?:No aws folder provided} ARN_FILE=".arn-file.md" { - echo "### Elastic APM Python agent layer ARNs" + echo "
" + echo "Elastic APM Python agent layer ARNs" echo '' echo '|Region|ARN|' echo '|------|---|' @@ -22,4 +23,8 @@ for f in $(ls "${AWS_FOLDER}"); do echo "|${f}|${LAYER_VERSION_ARN}|" >> "${ARN_FILE}" done -echo '' >> "${ARN_FILE}" +{ + echo '' + echo '
' + echo '' +} >> "${ARN_FILE}" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c58fbb7c6..687ac5efb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -198,4 +198,4 @@ If you have commit access, the process is as follows: 1. After tests pass, Github Actions will automatically build and push the new release to PyPI. 1. Edit and publish the [draft Github release](https://github.com/elastic/apm-agent-python/releases) created by Github Actions. Substitute the generated changelog with one hand written into the body of the - release and move the agent layer ARNs under a `
` block with a `summary`. + release. From 6e72b029ee4ede3bf3ae69334d6b5f859802d876 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Tue, 25 Jun 2024 20:53:03 +0200 Subject: [PATCH 088/103] Add necessary permissions to use aws/auth action (#2072) --- .github/workflows/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1fd242f5..6759af036 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,6 +71,9 @@ jobs: subject-path: "${{ github.workspace }}/build/dist/elastic-apm-python-lambda-layer.zip" publish-lambda-layers: + permissions: + contents: read + id-token: write needs: - build-distribution runs-on: ubuntu-latest From 183c0c4dcee8358824cfaa2e94b74ede15958e3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 09:34:51 +0200 Subject: [PATCH 089/103] build(deps): bump docker/build-push-action in the github-actions group (#2074) Bumps the github-actions group with 1 update: [docker/build-push-action](https://github.com/docker/build-push-action). Updates `docker/build-push-action` from 6.1.0 to 6.2.0 - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/31159d49c0d4756269a0940a750801a1ea5d7003...15560696de535e4014efeff63c48f16952e52dd1) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6759af036..8c6af2473 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -143,7 +143,7 @@ jobs: - name: Build and push image id: push - uses: docker/build-push-action@31159d49c0d4756269a0940a750801a1ea5d7003 # v6.1.0 + uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1 # v6.2.0 with: context: . push: true From 9d81d92916d68918662779e97cb079afb01a42ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 08:42:44 +0200 Subject: [PATCH 090/103] build(deps): bump docker/build-push-action in the github-actions group (#2079) Bumps the github-actions group with 1 update: [docker/build-push-action](https://github.com/docker/build-push-action). Updates `docker/build-push-action` from 6.2.0 to 6.3.0 - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/15560696de535e4014efeff63c48f16952e52dd1...1a162644f9a7e87d8f4b053101d1d9a712edc18c) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8c6af2473..33a54e798 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -143,7 +143,7 @@ jobs: - name: Build and push image id: push - uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1 # v6.2.0 + uses: docker/build-push-action@1a162644f9a7e87d8f4b053101d1d9a712edc18c # v6.3.0 with: context: . push: true From 91a53fc7ba192272a797706770b9c63df2376b2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 08:43:26 +0200 Subject: [PATCH 091/103] build(deps): bump certifi from 2024.6.2 to 2024.7.4 in /dev-utils (#2078) Bumps [certifi](https://github.com/certifi/python-certifi) from 2024.6.2 to 2024.7.4. - [Commits](https://github.com/certifi/python-certifi/compare/2024.06.02...2024.07.04) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dev-utils/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-utils/requirements.txt b/dev-utils/requirements.txt index 87ab0cf28..ccc3d9baf 100644 --- a/dev-utils/requirements.txt +++ b/dev-utils/requirements.txt @@ -1,4 +1,4 @@ # These are the pinned requirements for the lambda layer/docker image -certifi==2024.6.2 +certifi==2024.7.4 urllib3==1.26.19 wrapt==1.14.1 From 787a77a793dc4e7c8eafd5168f9771cab5d02d0a Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 9 Jul 2024 10:00:39 +0200 Subject: [PATCH 092/103] instrumentation/kafka: fix handling consumer iteration if transaction not sampled (#2075) Handle the case where if the transaction is not sampled capture_span will return None instead of span. While at it fix handling of checking for KAFKA_HOST in tests. Fix #2073 --- elasticapm/instrumentation/packages/kafka.py | 3 ++- tests/instrumentation/kafka_tests.py | 24 +++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/elasticapm/instrumentation/packages/kafka.py b/elasticapm/instrumentation/packages/kafka.py index c3bc2d64d..ab9ebd1a4 100644 --- a/elasticapm/instrumentation/packages/kafka.py +++ b/elasticapm/instrumentation/packages/kafka.py @@ -143,7 +143,8 @@ def call(self, module, method, wrapped, instance, args, kwargs): try: result = wrapped(*args, **kwargs) except StopIteration: - span.cancel() + if span: + span.cancel() raise if span and not isinstance(span, DroppedSpan): topic = result[0] diff --git a/tests/instrumentation/kafka_tests.py b/tests/instrumentation/kafka_tests.py index 71416c130..0bfc5c496 100644 --- a/tests/instrumentation/kafka_tests.py +++ b/tests/instrumentation/kafka_tests.py @@ -45,11 +45,10 @@ pytestmark = [pytest.mark.kafka] -if "KAFKA_HOST" not in os.environ: +KAFKA_HOST = os.environ.get("KAFKA_HOST") +if not KAFKA_HOST: pytestmark.append(pytest.mark.skip("Skipping kafka tests, no KAFKA_HOST environment variable set")) -KAFKA_HOST = os.environ["KAFKA_HOST"] - @pytest.fixture(scope="function") def topics(): @@ -233,3 +232,22 @@ def test_kafka_poll_unsampled_transaction(instrument, elasticapm_client, consume elasticapm_client.end_transaction("foo") spans = elasticapm_client.events[SPAN] assert len(spans) == 0 + + +def test_kafka_consumer_unsampled_transaction_handles_stop_iteration( + instrument, elasticapm_client, producer, consumer, topics +): + def delayed_send(): + time.sleep(0.2) + producer.send("test", key=b"foo", value=b"bar") + + thread = threading.Thread(target=delayed_send) + thread.start() + transaction = elasticapm_client.begin_transaction("foo") + transaction.is_sampled = False + for item in consumer: + pass + thread.join() + elasticapm_client.end_transaction("foo") + spans = elasticapm_client.events[SPAN] + assert len(spans) == 0 From bb88201c36d4aa7c37ee0fac0de3cdf612c810e6 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 9 Jul 2024 10:01:05 +0200 Subject: [PATCH 093/103] Promote some PendingDeprecationWarning to DeprecationWarning (#2076) Promote the removal of opentracing integration and the removal of logging handling to DeprecationWarnings. Refs #2029 --- elasticapm/contrib/django/handlers.py | 4 ++-- elasticapm/contrib/flask/__init__.py | 2 +- elasticapm/contrib/opentracing/__init__.py | 4 ++-- elasticapm/handlers/logging.py | 11 ++++------- tests/handlers/logging/logging_tests.py | 2 +- 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/elasticapm/contrib/django/handlers.py b/elasticapm/contrib/django/handlers.py index 0c97e888d..c980acc4f 100644 --- a/elasticapm/contrib/django/handlers.py +++ b/elasticapm/contrib/django/handlers.py @@ -47,11 +47,11 @@ class LoggingHandler(BaseLoggingHandler): def __init__(self, level=logging.NOTSET) -> None: warnings.warn( - "The LoggingHandler will be deprecated in v7.0 of the agent. " + "The LoggingHandler is deprecated and will be removed in v7.0 of the agent. " "Please use `log_ecs_reformatting` and ship the logs with Elastic " "Agent or Filebeat instead. " "https://www.elastic.co/guide/en/apm/agent/python/current/logs.html", - PendingDeprecationWarning, + DeprecationWarning, ) # skip initialization of BaseLoggingHandler logging.Handler.__init__(self, level=level) diff --git a/elasticapm/contrib/flask/__init__.py b/elasticapm/contrib/flask/__init__.py index e9ec3323e..fdb6906dd 100644 --- a/elasticapm/contrib/flask/__init__.py +++ b/elasticapm/contrib/flask/__init__.py @@ -87,7 +87,7 @@ def __init__(self, app=None, client=None, client_cls=Client, logging=False, **de if self.logging: warnings.warn( "Flask log shipping is deprecated. See the Flask docs for more info and alternatives.", - PendingDeprecationWarning, + DeprecationWarning, ) self.client = client or get_client() self.client_cls = client_cls diff --git a/elasticapm/contrib/opentracing/__init__.py b/elasticapm/contrib/opentracing/__init__.py index 8fbc99b19..71619ea20 100644 --- a/elasticapm/contrib/opentracing/__init__.py +++ b/elasticapm/contrib/opentracing/__init__.py @@ -36,8 +36,8 @@ warnings.warn( ( - "The OpenTracing bridge will be deprecated in the next major release. " + "The OpenTracing bridge is deprecated and will be removed in the next major release. " "Please migrate to the OpenTelemetry bridge." ), - PendingDeprecationWarning, + DeprecationWarning, ) diff --git a/elasticapm/handlers/logging.py b/elasticapm/handlers/logging.py index ed4db87ac..96718d2db 100644 --- a/elasticapm/handlers/logging.py +++ b/elasticapm/handlers/logging.py @@ -51,7 +51,7 @@ def __init__(self, *args, **kwargs) -> None: "the agent. Please use `log_ecs_reformatting` and ship the logs " "with Elastic Agent or Filebeat instead. " "https://www.elastic.co/guide/en/apm/agent/python/current/logs.html", - PendingDeprecationWarning, + DeprecationWarning, ) self.client = None if "client" in kwargs: @@ -66,12 +66,9 @@ def __init__(self, *args, **kwargs) -> None: if client_cls: self.client = client_cls(*args, **kwargs) else: - # In 6.0, this should raise a ValueError warnings.warn( - "LoggingHandler requires a Client instance. No Client was " - "received. This will result in an error starting in v6.0 " - "of the agent", - PendingDeprecationWarning, + "LoggingHandler requires a Client instance. No Client was received.", + DeprecationWarning, ) self.client = Client(*args, **kwargs) logging.Handler.__init__(self, level=kwargs.get("level", logging.NOTSET)) @@ -201,7 +198,7 @@ def __init__(self, name=""): "the agent. On Python 3.2+, by default we add a LogRecordFactory to " "your root logger automatically" "https://www.elastic.co/guide/en/apm/agent/python/current/logs.html", - PendingDeprecationWarning, + DeprecationWarning, ) def filter(self, record): diff --git a/tests/handlers/logging/logging_tests.py b/tests/handlers/logging/logging_tests.py index 8e23a0b69..8cc8fc4f1 100644 --- a/tests/handlers/logging/logging_tests.py +++ b/tests/handlers/logging/logging_tests.py @@ -380,7 +380,7 @@ def test_logging_handler_no_client(recwarn): while True: # If we never find our desired warning this will eventually throw an # AssertionError - w = recwarn.pop(PendingDeprecationWarning) + w = recwarn.pop(DeprecationWarning) if "LoggingHandler requires a Client instance" in w.message.args[0]: return True From 49852a47c5d0768ea5671f453ac80dbfdcfc950a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 9 Jul 2024 10:02:31 +0200 Subject: [PATCH 094/103] updatecli: use shared policy (#2077) --- .../updatecli.d/update-gherkin-specs.yml | 82 ------------------ .../updatecli.d/update-json-specs.yml | 82 ------------------ .ci/updatecli/updatecli.d/update-specs.yml | 84 ------------------- .ci/updatecli/values.d/apm-data-spec.yml | 1 + .ci/updatecli/values.d/apm-gherkin.yml | 1 + .ci/updatecli/values.d/apm-json-specs.yml | 1 + .ci/updatecli/values.d/scm.yml | 7 ++ .ci/updatecli/values.yml | 14 ---- .github/workflows/updatecli.yml | 19 ++++- update-compose.yaml | 18 ++++ 10 files changed, 45 insertions(+), 264 deletions(-) delete mode 100644 .ci/updatecli/updatecli.d/update-gherkin-specs.yml delete mode 100644 .ci/updatecli/updatecli.d/update-json-specs.yml delete mode 100644 .ci/updatecli/updatecli.d/update-specs.yml create mode 100644 .ci/updatecli/values.d/apm-data-spec.yml create mode 100644 .ci/updatecli/values.d/apm-gherkin.yml create mode 100644 .ci/updatecli/values.d/apm-json-specs.yml create mode 100644 .ci/updatecli/values.d/scm.yml delete mode 100644 .ci/updatecli/values.yml create mode 100644 update-compose.yaml diff --git a/.ci/updatecli/updatecli.d/update-gherkin-specs.yml b/.ci/updatecli/updatecli.d/update-gherkin-specs.yml deleted file mode 100644 index 82f986fae..000000000 --- a/.ci/updatecli/updatecli.d/update-gherkin-specs.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: update-gherkin-specs -pipelineid: update-gherkin-specs - -scms: - default: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - apm: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.apm_repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - -sources: - sha: - kind: file - spec: - file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ .github.branch }}.patch' - matchpattern: "^From\\s([0-9a-f]{40})\\s" - transformers: - - findsubmatch: - pattern: "[0-9a-f]{40}" - pull_request: - kind: shell - dependson: - - sha - spec: - command: gh api /repos/{{ .github.owner }}/{{ .github.apm_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' - environments: - - name: GITHUB_TOKEN - - name: PATH - agents-gherkin-specs-tarball: - kind: shell - scmid: apm - dependson: - - sha - spec: - command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/gherkin-specs.tgz . - environments: - - name: PATH - workdir: "{{ .specs.apm_gherkin_path }}" - -actions: - pr: - kind: "github/pullrequest" - scmid: default - spec: - automerge: false - draft: false - labels: - - "automation" - description: |- - ### What - APM agent Gherkin specs automatic sync - - ### Why - *Changeset* - * {{ source "pull_request" }} - * https://github.com/elastic/apm/commit/{{ source "sha" }} - title: '[Automation] Update Gherkin specs' - -targets: - agent-gherkin-specs: - name: APM agent gherkin specs {{ source "sha" }} - scmid: default - disablesourceinput: true - kind: shell - spec: - # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. - # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target - command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/gherkin-specs.tgz && git --no-pager diff' - workdir: "{{ .apm_agent.gherkin_specs_path }}" diff --git a/.ci/updatecli/updatecli.d/update-json-specs.yml b/.ci/updatecli/updatecli.d/update-json-specs.yml deleted file mode 100644 index 7b86367a7..000000000 --- a/.ci/updatecli/updatecli.d/update-json-specs.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: update-json-specs -pipelineid: update-json-specs - -scms: - default: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - apm: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.apm_repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - -sources: - sha: - kind: file - spec: - file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ .github.branch }}.patch' - matchpattern: "^From\\s([0-9a-f]{40})\\s" - transformers: - - findsubmatch: - pattern: "[0-9a-f]{40}" - pull_request: - kind: shell - dependson: - - sha - spec: - command: gh api /repos/{{ .github.owner }}/{{ .github.apm_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' - environments: - - name: GITHUB_TOKEN - - name: PATH - agents-json-specs-tarball: - kind: shell - scmid: apm - dependson: - - sha - spec: - command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-specs.tgz . - environments: - - name: PATH - workdir: "{{ .specs.apm_json_path }}" - -actions: - pr: - kind: "github/pullrequest" - scmid: default - spec: - automerge: false - draft: false - labels: - - "automation" - description: |- - ### What - APM agent specs automatic sync - - ### Why - *Changeset* - * {{ source "pull_request" }} - * https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ source "sha" }} - title: '[Automation] Update JSON specs' - -targets: - agent-json-specs: - name: APM agent json specs {{ source "sha" }} - scmid: default - disablesourceinput: true - kind: shell - spec: - # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. - # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target - command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-specs.tgz && git --no-pager diff' - workdir: "{{ .apm_agent.json_specs_path }}" diff --git a/.ci/updatecli/updatecli.d/update-specs.yml b/.ci/updatecli/updatecli.d/update-specs.yml deleted file mode 100644 index ab5589f7e..000000000 --- a/.ci/updatecli/updatecli.d/update-specs.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: update-specs -pipelineid: update-schema-specs - -scms: - default: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - - apm-data: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.apm_data_repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - -sources: - sha: - kind: file - spec: - file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_data_repository }}/commit/{{ .github.branch }}.patch' - matchpattern: "^From\\s([0-9a-f]{40})\\s" - transformers: - - findsubmatch: - pattern: "[0-9a-f]{40}" - pull_request: - kind: shell - dependson: - - sha - spec: - command: gh api /repos/{{ .github.owner }}/{{ .github.apm_data_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' - environments: - - name: GITHUB_TOKEN - - name: PATH - agent-specs-tarball: - kind: shell - scmid: apm-data - dependson: - - sha - spec: - command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-schema.tgz . - environments: - - name: PATH - workdir: "{{ .specs.apm_data_path }}" - -actions: - pr: - kind: "github/pullrequest" - scmid: default - sourceid: sha - spec: - automerge: false - draft: false - labels: - - "automation" - description: |- - ### What - APM agent json server schema automatic sync - - ### Why - *Changeset* - * {{ source "pull_request" }} - * https://github.com/{{ .github.owner }}/{{ .github.apm_data_repository }}/commit/{{ source "sha" }} - title: '[Automation] Update JSON server schema specs' - -targets: - agent-json-schema: - name: APM agent json server schema {{ source "sha" }} - scmid: default - disablesourceinput: true - kind: shell - spec: - # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. - # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target - command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-schema.tgz && git --no-pager diff' - workdir: "{{ .apm_agent.server_schema_specs_path }}" diff --git a/.ci/updatecli/values.d/apm-data-spec.yml b/.ci/updatecli/values.d/apm-data-spec.yml new file mode 100644 index 000000000..4bf89f633 --- /dev/null +++ b/.ci/updatecli/values.d/apm-data-spec.yml @@ -0,0 +1 @@ +apm_schema_specs_path: tests/upstream/json-specs diff --git a/.ci/updatecli/values.d/apm-gherkin.yml b/.ci/updatecli/values.d/apm-gherkin.yml new file mode 100644 index 000000000..7234fe8c8 --- /dev/null +++ b/.ci/updatecli/values.d/apm-gherkin.yml @@ -0,0 +1 @@ +apm_gherkin_specs_path: tests/bdd/features \ No newline at end of file diff --git a/.ci/updatecli/values.d/apm-json-specs.yml b/.ci/updatecli/values.d/apm-json-specs.yml new file mode 100644 index 000000000..c527210e4 --- /dev/null +++ b/.ci/updatecli/values.d/apm-json-specs.yml @@ -0,0 +1 @@ +apm_json_specs_path: tests/upstream/json-specs diff --git a/.ci/updatecli/values.d/scm.yml b/.ci/updatecli/values.d/scm.yml new file mode 100644 index 000000000..78f9e4bc0 --- /dev/null +++ b/.ci/updatecli/values.d/scm.yml @@ -0,0 +1,7 @@ +scm: + enabled: true + owner: elastic + repository: apm-agent-python + branch: main + +signedcommit: true \ No newline at end of file diff --git a/.ci/updatecli/values.yml b/.ci/updatecli/values.yml deleted file mode 100644 index b0b58d73e..000000000 --- a/.ci/updatecli/values.yml +++ /dev/null @@ -1,14 +0,0 @@ -github: - owner: "elastic" - repository: "apm-agent-python" - apm_repository: "apm" - apm_data_repository: "apm-data" - branch: "main" -specs: - apm_data_path: "input/elasticapm/docs/spec/v2" - apm_json_path: "tests/agents/json-specs" - apm_gherkin_path: "tests/agents/gherkin-specs" -apm_agent: - gherkin_specs_path: "tests/bdd/features" - json_specs_path: "tests/upstream/json-specs" - server_schema_specs_path: "tests/upstream/json-specs" \ No newline at end of file diff --git a/.github/workflows/updatecli.yml b/.github/workflows/updatecli.yml index 8190487ad..b2d1e7a8a 100644 --- a/.github/workflows/updatecli.yml +++ b/.github/workflows/updatecli.yml @@ -9,14 +9,29 @@ permissions: contents: read jobs: - bump: + compose: runs-on: ubuntu-latest + permissions: + contents: read + packages: read steps: - uses: actions/checkout@v4 + - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: --experimental compose diff + env: + GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }} + - uses: elastic/oblt-actions/updatecli/run@v1 with: - command: "apply --config .ci/updatecli/updatecli.d --values .ci/updatecli/values.yml" + command: --experimental compose apply env: GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }} diff --git a/update-compose.yaml b/update-compose.yaml new file mode 100644 index 000000000..d40020933 --- /dev/null +++ b/update-compose.yaml @@ -0,0 +1,18 @@ +policies: + - name: Handle apm-data server specs + policy: ghcr.io/elastic/oblt-updatecli-policies/apm/apm-data-spec:0.2.0@sha256:7069c0773d44a74c4c8103b4d9957b468f66081ee9d677238072fe11c4d2197c + values: + - .ci/updatecli/values.d/scm.yml + - .ci/updatecli/values.d/apm-data-spec.yml + + - name: Handle apm gherkin specs + policy: ghcr.io/elastic/oblt-updatecli-policies/apm/apm-gherkin:0.2.0@sha256:26a30ad2b98a6e4cb17fb88a28fa3277ced8ca862d6388943afaafbf8ee96e7d + values: + - .ci/updatecli/values.d/scm.yml + - .ci/updatecli/values.d/apm-gherkin.yml + + - name: Handle apm json specs + policy: ghcr.io/elastic/oblt-updatecli-policies/apm/apm-json-specs:0.2.0@sha256:969a6d21eabd6ebea66cb29b35294a273d6dbc0f7da78589c416aedf08728e78 + values: + - .ci/updatecli/values.d/scm.yml + - .ci/updatecli/values.d/apm-json-specs.yml From 3b32ee652e51226f002b9834508be790c8dd8b54 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 10 Jul 2024 10:23:01 +0200 Subject: [PATCH 095/103] feat: make published Docker images multi-platform, add linux/arm64 plat (#2080) --- .github/workflows/release.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33a54e798..ea2ab7b78 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -118,6 +118,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + - name: Log in to the Elastic Container registry uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 with: @@ -142,10 +145,11 @@ jobs: suffix=${{ contains(matrix.dockerfile, 'wolfi') && '-wolfi' || '' }} - name: Build and push image - id: push + id: docker-push uses: docker/build-push-action@1a162644f9a7e87d8f4b053101d1d9a712edc18c # v6.3.0 with: context: . + platforms: linux/amd64,linux/arm64 push: true file: ${{ matrix.dockerfile }} tags: ${{ steps.docker-meta.outputs.tags }} @@ -157,7 +161,7 @@ jobs: uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 with: subject-name: "${{ env.DOCKER_IMAGE_NAME }}" - subject-digest: ${{ steps.push.outputs.digest }} + subject-digest: ${{ steps.docker-push.outputs.digest }} push-to-registry: true github-draft: @@ -196,7 +200,7 @@ jobs: with: needs: ${{ toJSON(needs) }} - if: startsWith(github.ref, 'refs/tags') - uses: elastic/oblt-actions/slack/notify-result@v1.7.0 + uses: elastic/oblt-actions/slack/notify-result@v1 with: bot-token: ${{ secrets.SLACK_BOT_TOKEN }} channel-id: "#apm-agent-python" From bc2ca46cec45a12cea4d4d164c1610b6cc793a06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 09:22:17 +0200 Subject: [PATCH 096/103] build(deps): bump the github-actions group with 2 updates (#2081) Bumps the github-actions group with 2 updates: [actions/attest-build-provenance](https://github.com/actions/attest-build-provenance) and [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action). Updates `actions/attest-build-provenance` from 1.3.2 to 1.3.3 - [Release notes](https://github.com/actions/attest-build-provenance/releases) - [Changelog](https://github.com/actions/attest-build-provenance/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest-build-provenance/compare/bdd51370e0416ac948727f861e03c2f05d32d78e...5e9cb68e95676991667494a6a4e59b8a2f13e1d0) Updates `docker/setup-buildx-action` from 3.3.0 to 3.4.0 - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/d70bba72b1f3fd22344832f00baa16ece964efeb...4fd812986e6c8c2a69e18311145f9371337f27d4) --- updated-dependencies: - dependency-name: actions/attest-build-provenance dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea2ab7b78..08457b92e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/packages - name: generate build provenance - uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + uses: actions/attest-build-provenance@5e9cb68e95676991667494a6a4e59b8a2f13e1d0 # v1.3.3 with: subject-path: "${{ github.workspace }}/dist/*" @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/build-distribution - name: generate build provenance - uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + uses: actions/attest-build-provenance@5e9cb68e95676991667494a6a4e59b8a2f13e1d0 # v1.3.3 with: subject-path: "${{ github.workspace }}/build/dist/elastic-apm-python-lambda-layer.zip" @@ -119,7 +119,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v3.4.0 - name: Log in to the Elastic Container registry uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 @@ -158,7 +158,7 @@ jobs: AGENT_DIR=./build/dist/package/python - name: generate build provenance (containers) - uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + uses: actions/attest-build-provenance@5e9cb68e95676991667494a6a4e59b8a2f13e1d0 # v1.3.3 with: subject-name: "${{ env.DOCKER_IMAGE_NAME }}" subject-digest: ${{ steps.docker-push.outputs.digest }} From dcbd86e7b4b5051b7ecb418d39bbaf9f2939f1b0 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 17 Jul 2024 17:38:41 +0200 Subject: [PATCH 097/103] contrib/serverless: capture_serverless is already called by the APM layer (#2083) Add a small note in capture_serverless docstring that with the APM layer it is already been called and adding it manually will result in instrumenting the handler twice. --- elasticapm/contrib/serverless/aws.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elasticapm/contrib/serverless/aws.py b/elasticapm/contrib/serverless/aws.py index 9f5f7b133..1717d57cc 100644 --- a/elasticapm/contrib/serverless/aws.py +++ b/elasticapm/contrib/serverless/aws.py @@ -71,6 +71,9 @@ def capture_serverless(func: Optional[callable] = None, **kwargs) -> callable: @capture_serverless def handler(event, context): return {"statusCode": r.status_code, "body": "Success!"} + + Please note that when using the APM Layer and setting AWS_LAMBDA_EXEC_WRAPPER this is not required and + the handler would be instrumented automatically. """ if not func: # This allows for `@capture_serverless()` in addition to From 284d73f19ccd3b87da0d61447ae45f54a4f38a9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:03:04 +0200 Subject: [PATCH 098/103] build(deps): bump docker/build-push-action in the github-actions group (#2086) Bumps the github-actions group with 1 update: [docker/build-push-action](https://github.com/docker/build-push-action). Updates `docker/build-push-action` from 6.3.0 to 6.4.1 - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/1a162644f9a7e87d8f4b053101d1d9a712edc18c...1ca370b3a9802c92e886402e0dd88098a2533b12) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 08457b92e..8148d1548 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -146,7 +146,7 @@ jobs: - name: Build and push image id: docker-push - uses: docker/build-push-action@1a162644f9a7e87d8f4b053101d1d9a712edc18c # v6.3.0 + uses: docker/build-push-action@1ca370b3a9802c92e886402e0dd88098a2533b12 # v6.4.1 with: context: . platforms: linux/amd64,linux/arm64 From a8213f481d2331ef4c749d3c09fd528fd15641d3 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 26 Jul 2024 09:13:24 +0200 Subject: [PATCH 099/103] elasticapm/transport: specific shutdown handling for http transport (#2085) We have a race condition at http transport shutdown where our atexit handler is racing against urllib3 ConnectionPool weakref finalizer. Having the urllib3 finalizer called before atexit would lead to have our thread hang waiting for send any eventual queued data via urllib3 pools that are closed. Force the creation of a new PoolManager so that we are always able to flush. --- elasticapm/transport/base.py | 3 +- elasticapm/transport/http.py | 18 +++++++ tests/transports/test_urllib3.py | 88 ++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) diff --git a/elasticapm/transport/base.py b/elasticapm/transport/base.py index 487e13a93..24911c395 100644 --- a/elasticapm/transport/base.py +++ b/elasticapm/transport/base.py @@ -292,7 +292,8 @@ def close(self) -> None: if not self._flushed.wait(timeout=self._max_flush_time_seconds): logger.error("Closing the transport connection timed out.") - stop_thread = close + def stop_thread(self) -> None: + self.close() def flush(self): """ diff --git a/elasticapm/transport/http.py b/elasticapm/transport/http.py index 17ae50ba3..ed132068c 100644 --- a/elasticapm/transport/http.py +++ b/elasticapm/transport/http.py @@ -32,6 +32,7 @@ import hashlib import json +import os import re import ssl import urllib.parse @@ -250,6 +251,23 @@ def ca_certs(self): return self._server_ca_cert_file return certifi.where() if (certifi and self.client.config.use_certifi) else None + def close(self): + """ + Take care of being able to shutdown cleanly + :return: + """ + if self._closed or (not self._thread or self._thread.pid != os.getpid()): + return + + self._closed = True + # we are racing against urllib3 ConnectionPool weakref finalizer that would lead to having them closed + # and we hanging waiting for send any eventual queued data + # Force the creation of a new PoolManager so that we are always able to flush + self._http = None + self.queue("close", None) + if not self._flushed.wait(timeout=self._max_flush_time_seconds): + logger.error("Closing the transport connection timed out.") + def version_string_to_tuple(version): if version: diff --git a/tests/transports/test_urllib3.py b/tests/transports/test_urllib3.py index b24408e54..32a5b7384 100644 --- a/tests/transports/test_urllib3.py +++ b/tests/transports/test_urllib3.py @@ -30,6 +30,7 @@ import os +import time import certifi import mock @@ -517,3 +518,90 @@ def test_fetch_server_info_flat_string(waiting_httpserver, caplog, elasticapm_cl transport.fetch_server_info() assert elasticapm_client.server_version is None assert_any_record_contains(caplog.records, "No version key found in server response") + + +def test_close(waiting_httpserver, elasticapm_client): + elasticapm_client.server_version = (8, 0, 0) # avoid making server_info request + waiting_httpserver.serve_content(code=202, content="", headers={"Location": "http://example.com/foo"}) + transport = Transport( + waiting_httpserver.url, client=elasticapm_client, headers=elasticapm_client._transport._headers + ) + transport.start_thread() + + transport.close() + + assert transport._closed is True + assert transport._flushed.is_set() is True + + +def test_close_does_nothing_if_called_from_another_pid(waiting_httpserver, caplog, elasticapm_client): + elasticapm_client.server_version = (8, 0, 0) # avoid making server_info request + waiting_httpserver.serve_content(code=202, content="", headers={"Location": "http://example.com/foo"}) + transport = Transport( + waiting_httpserver.url, client=elasticapm_client, headers=elasticapm_client._transport._headers + ) + transport.start_thread() + + with mock.patch("os.getpid") as getpid_mock: + getpid_mock.return_value = 0 + transport.close() + + assert transport._closed is False + + transport.close() + + +def test_close_can_be_called_multiple_times(waiting_httpserver, caplog, elasticapm_client): + elasticapm_client.server_version = (8, 0, 0) # avoid making server_info request + waiting_httpserver.serve_content(code=202, content="", headers={"Location": "http://example.com/foo"}) + transport = Transport( + waiting_httpserver.url, client=elasticapm_client, headers=elasticapm_client._transport._headers + ) + transport.start_thread() + + with caplog.at_level("INFO", logger="elasticapm.transport.http"): + transport.close() + + assert transport._closed is True + + transport.close() + + +def test_close_timeout_error_without_flushing(waiting_httpserver, caplog, elasticapm_client): + elasticapm_client.server_version = (8, 0, 0) # avoid making server_info request + waiting_httpserver.serve_content(code=202, content="", headers={"Location": "http://example.com/foo"}) + + with caplog.at_level("INFO", logger="elasticapm.transport.http"): + with mock.patch.object(Transport, "_max_flush_time_seconds", 0): + with mock.patch.object(Transport, "_flush") as flush_mock: + # sleep more that the timeout + flush_mock.side_effect = lambda x: time.sleep(0.1) + transport = Transport( + waiting_httpserver.url, client=elasticapm_client, headers=elasticapm_client._transport._headers + ) + transport.start_thread() + # need to write something to the buffer to have _flush() called + transport.queue("error", {"an": "error"}) + transport.close() + + assert transport._flushed.is_set() is False + assert transport._closed is True + record = caplog.records[-1] + assert "Closing the transport connection timed out." in record.msg + + +def test_http_pool_manager_is_recycled_at_stop_thread(waiting_httpserver, caplog, elasticapm_client): + elasticapm_client.server_version = (8, 0, 0) # avoid making server_info request + waiting_httpserver.serve_content(code=202, content="", headers={"Location": "http://example.com/foo"}) + transport = Transport( + waiting_httpserver.url, client=elasticapm_client, headers=elasticapm_client._transport._headers + ) + transport.start_thread() + pool_manager = transport.http + + with caplog.at_level("INFO", logger="elasticapm.transport.http"): + transport.stop_thread() + + assert transport._flushed.is_set() is True + assert pool_manager != transport._http + assert not caplog.records From fde6c96648cc589c6f39f9b80d4a468cdfe71783 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Mon, 29 Jul 2024 09:53:46 +0200 Subject: [PATCH 100/103] tests: bump setuptools to latest (#2087) For some reason twisted tests are now failing with not so recent setuptools versions: 56.0 fails but 59.6.0 works. Twisted fails to install with: File "/usr/local/lib/python3.8/site-packages/pkg_resources/__init__.py", line 2456, in resolve module = __import__(self.module_name, fromlist=['__name__'], level=0) ModuleNotFoundError: No module named 'setuptools.command.build' --- tests/scripts/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/run_tests.sh b/tests/scripts/run_tests.sh index fc248949f..7fcc85010 100755 --- a/tests/scripts/run_tests.sh +++ b/tests/scripts/run_tests.sh @@ -3,7 +3,7 @@ set -e export PATH=${HOME}/.local/bin:${PATH} -python -m pip install --user -U pip --cache-dir "${PIP_CACHE}" +python -m pip install --user -U pip setuptools --cache-dir "${PIP_CACHE}" python -m pip install --user -r "tests/requirements/reqs-${FRAMEWORK}.txt" --cache-dir "${PIP_CACHE}" export PYTHON_VERSION=$(python -c "import platform; pv=platform.python_version_tuple(); print('pypy' + ('' if pv[0] == 2 else str(pv[0])) if platform.python_implementation() == 'PyPy' else '.'.join(map(str, platform.python_version_tuple()[:2])))") From efab156183c92b1baf42007bc181d78073309136 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:42:10 +0200 Subject: [PATCH 101/103] build(deps): bump the github-actions group with 3 updates (#2089) Bumps the github-actions group with 3 updates: [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action), [docker/login-action](https://github.com/docker/login-action) and [docker/build-push-action](https://github.com/docker/build-push-action). Updates `docker/setup-buildx-action` from 3.4.0 to 3.5.0 - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/4fd812986e6c8c2a69e18311145f9371337f27d4...aa33708b10e362ff993539393ff100fa93ed6a27) Updates `docker/login-action` from 3.2.0 to 3.3.0 - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/0d4c9c5ea7693da7b068278f7b52bda2a190a446...9780b0c442fbb1117ed29e0efdff1e18412f7567) Updates `docker/build-push-action` from 6.4.1 to 6.5.0 - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/1ca370b3a9802c92e886402e0dd88098a2533b12...5176d81f87c23d6fc96624dfdbcd9f3830bbe445) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 6 +++--- .github/workflows/updatecli.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8148d1548..a68e508c3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,10 +119,10 @@ jobs: - uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v3.4.0 + uses: docker/setup-buildx-action@aa33708b10e362ff993539393ff100fa93ed6a27 # v3.5.0 - name: Log in to the Elastic Container registry - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }} username: ${{ secrets.ELASTIC_DOCKER_USERNAME }} @@ -146,7 +146,7 @@ jobs: - name: Build and push image id: docker-push - uses: docker/build-push-action@1ca370b3a9802c92e886402e0dd88098a2533b12 # v6.4.1 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v6.5.0 with: context: . platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/updatecli.yml b/.github/workflows/updatecli.yml index b2d1e7a8a..c56645f0e 100644 --- a/.github/workflows/updatecli.yml +++ b/.github/workflows/updatecli.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ghcr.io username: ${{ github.actor }} From 566ae62750154ba7860cd03796af1b6fddfcb25f Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 30 Jul 2024 09:47:22 +0200 Subject: [PATCH 102/103] Drop `setup.py test` (#2090) * scripts/run-tests: remove python2 relics * setup.py: drop test For compatibility with setuptools>=72 that remove the test command. See https://github.com/pypa/setuptools/issues/931 * setup.py: remove unused imports --- scripts/run-tests.bat | 6 ++---- setup.py | 38 ++------------------------------------ 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/scripts/run-tests.bat b/scripts/run-tests.bat index 0fce63000..f17e57d9c 100644 --- a/scripts/run-tests.bat +++ b/scripts/run-tests.bat @@ -9,14 +9,12 @@ set VENV_PYTHON=%cd%\venv\Scripts\ set COVERAGE_FILE=.coverage.windows.%VERSION%.%FRAMEWORK%.%ASYNCIO% -set IGNORE_PYTHON3_WITH_PYTHON2= -if "%VERSION%" == "2.7" set IGNORE_PYTHON3_WITH_PYTHON2=--ignore-glob="*\py3_*.py" set PYTEST_JUNIT="--junitxml=.\tests\windows-%VERSION%-%FRAMEWORK%-%ASYNCIO%-python-agent-junit.xml" if "%ASYNCIO%" == "true" ( - %VENV_PYTHON%\python.exe -m pytest %PYTEST_JUNIT% %IGNORE_PYTHON3_WITH_PYTHON2% --cov --cov-context=test --cov-branch --cov-config=setup.cfg -m "not integrationtest" || exit /b 1 + %VENV_PYTHON%\python.exe -m pytest %PYTEST_JUNIT% --cov --cov-context=test --cov-branch --cov-config=setup.cfg -m "not integrationtest" || exit /b 1 ) if "%ASYNCIO%" == "false" ( - %VENV_PYTHON%\python.exe -m pytest %PYTEST_JUNIT% --ignore-glob="*\asyncio*\*" %IGNORE_PYTHON3_WITH_PYTHON2% --cov --cov-context=test --cov-branch --cov-config=setup.cfg -m "not integrationtest" || exit /b 1 + %VENV_PYTHON%\python.exe -m pytest %PYTEST_JUNIT% --ignore-glob="*\asyncio*\*" --cov --cov-context=test --cov-branch --cov-config=setup.cfg -m "not integrationtest" || exit /b 1 ) call %VENV_PYTHON%\python.exe setup.py bdist_wheel diff --git a/setup.py b/setup.py index a88cdeb5b..5dbb8f643 100644 --- a/setup.py +++ b/setup.py @@ -40,50 +40,16 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Hack to prevent stupid "TypeError: 'NoneType' object is not callable" error -# in multiprocessing/util.py _exit_function when running `python -# setup.py test` (see -# http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html) -for m in ("multiprocessing", "billiard"): - try: - __import__(m) - except ImportError: - pass - import ast import codecs import os -import sys -from distutils.command.build_ext import build_ext -from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatformError import pkg_resources -from setuptools import Extension, setup -from setuptools.command.test import test as TestCommand +from setuptools import setup pkg_resources.require("setuptools>=39.2") -class PyTest(TestCommand): - user_options = [("pytest-args=", "a", "Arguments to pass to py.test")] - - def initialize_options(self) -> None: - TestCommand.initialize_options(self) - self.pytest_args = [] - - def finalize_options(self) -> None: - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - def run_tests(self) -> None: - # import here, cause outside the eggs aren't loaded - import pytest - - errno = pytest.main(self.pytest_args) - sys.exit(errno) - - def get_version(): """ Get version without importing from elasticapm. This avoids any side effects @@ -108,4 +74,4 @@ def get_version(): return "unknown" -setup(cmdclass={"test": PyTest}, version=get_version()) +setup(version=get_version()) From aa2c0ff03ec24f7ad14532d363c758f780ec5904 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Tue, 30 Jul 2024 17:11:42 +0200 Subject: [PATCH 103/103] update CHANGELOG and bump version to 6.23.0 (#2088) --- CHANGELOG.asciidoc | 24 ++++++++++++++++++++++++ elasticapm/version.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index b8df53ab6..7550c42d6 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -32,6 +32,30 @@ endif::[] [[release-notes-6.x]] === Python Agent version 6.x +[[release-notes-6.23.0]] +==== 6.23.0 - 2024-07-30 + +[float] +===== Features + +* Make published Docker images multi-platform with the addition of linux/arm64 {pull}2080[#2080] + +[float] +===== Bug fixes + +* Fix handling consumer iteration if transaction not sampled in kafka instrumentation {pull}2075[#2075] +* Fix race condition with urllib3 at shutdown {pull}2085[#2085] +* Fix compatibility with setuptools>=72 that removed test command {pull}2090[#2090] + +===== Deprecations + +* Python 3.6 support will be removed in version 7.0.0 of the agent +* The log shipping LoggingHandler will be removed in version 7.0.0 of the agent. +* The log shipping feature in the Flask instrumentation will be removed in version 7.0.0 of the agent. +* The log shipping feature in the Django instrumentation will be removed in version 7.0.0 of the agent. +* The OpenTracing bridge will be removed in version 7.0.0 of the agent. +* Celery 4.0 support is deprecated because it's not installable anymore with a modern pip + [[release-notes-6.22.3]] ==== 6.22.3 - 2024-06-10 diff --git a/elasticapm/version.py b/elasticapm/version.py index 82aa446ad..e9eff8543 100644 --- a/elasticapm/version.py +++ b/elasticapm/version.py @@ -28,5 +28,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = (6, 22, 3) +__version__ = (6, 23, 0) VERSION = ".".join(map(str, __version__))