diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..e4e4859ac --- /dev/null +++ b/.flake8 @@ -0,0 +1,36 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# Unfortunately, flake8 does not support pyproject.toml configuration. +# https://github.com/PyCQA/flake8/issues/234 +[flake8] +# Disabling the following: +# E203: whitespace before ':'. Conflict with black. +# E266: too many leading '#' for block comment +# W503: line break before binary operator +# D105: Missing docstring in magic method +# D104: Missing docstring in public package +# D404: First word of the docstring should not be `This` +# PT009: use a regular assert instead of unittest-style +ignore = E203,E266,W503,D105,D404,PT009 +# Disabling the following for tests: +# D400: First line should end with a period +# D200: One-line docstring should fit on one line with quotes +# D102: Missing docstring in public method +# D104: Missing docstring in public package +# D107: Missing docstring in __init__ +per-file-ignores = + __init__.py:D104 + tests/*:D400,D200,D102,D104,D107 +max-line-length = 120 +show-source = true + +# Enable Bugbear's extended opinionated checks. +# https://github.com/PyCQA/flake8-bugbear#how-to-enable-opinionated-warnings +extend-select = B9 + +# Ensure that flake8 warnings are silenced correctly. +# https://github.com/plinss/flake8-noqa#options +noqa-require-code = true + +docstring-convention = numpy diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..f4afc6696 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,15 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# Set default behavior to automatically normalize line endings. +* text=auto + +# Force batch scripts to always use CRLF line endings so that if a repo is accessed +# in Windows via a file share from Linux, the scripts will work. +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf +*.{ps1,[pP][sS]1} text eol=crlf + +# Force bash scripts to always use LF line endings so that if a repo is accessed +# in Unix via a file share from Windows, the scripts will work. +*.sh text eol=lf diff --git a/.github/codeql/codeql-config.yaml b/.github/codeql/codeql-config.yaml new file mode 100644 index 000000000..3443adaf1 --- /dev/null +++ b/.github/codeql/codeql-config.yaml @@ -0,0 +1,6 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +name: CodeQL configuration +paths: +- src/macaron diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 000000000..444a20b4d --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,38 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# This configuration file enables Dependabot version updates. +# https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/about-dependabot-version-updates +# https://github.com/dependabot/feedback/issues/551 + +version: 2 +updates: +- package-ecosystem: pip + directory: / + schedule: + interval: weekly + commit-message: + prefix: chore + prefix-development: chore + include: scope + open-pull-requests-limit: 13 + target-branch: staging + # Add additional reviewers for PRs opened by Dependabot. For more information, see: + # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#reviewers + # reviewers: + # - + +- package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + commit-message: + prefix: chore + prefix-development: chore + include: scope + open-pull-requests-limit: 13 + target-branch: staging + # Add additional reviewers for PRs opened by Dependabot. For more information, see: + # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#reviewers + # reviewers: + # - diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml new file mode 100644 index 000000000..6f63131a5 --- /dev/null +++ b/.github/workflows/_build.yaml @@ -0,0 +1,134 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# This is a trusted builder implemented as a reusable workflow that can be called by other +# Actions workflows. It checks, tests, and builds the artifacts including SBOM and documentations, +# and computes hash digests as output to be used by a SLSA provenance generator. The artifacts are +# always uploaded for every job to be used for debugging purposes, but they will be removed within +# the specified retention days. +# +# Even though we run the build in a matrix to check against different platforms, due to a known +# limitation of reusable workflows that do not support setting strategy property from the caller +# workflow, we only generate artifacts for ubuntu-latest and Python 3.11, which can be used to +# create a release. For details see: +# +# https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations +# +# Note: if the build workflow needs to access secrets, they need to be passed by the caller using +# `secrets: inherit`. See also +# +# https://docs.github.com/en/actions/using-workflows/reusing-workflows +# https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions +# +# for the security recommendations. + +name: Build the package +on: + workflow_call: + outputs: + artifacts-sha256: + description: The hash of the artifacts + value: ${{ jobs.build.outputs.artifacts-sha256 }} +permissions: + contents: read +env: + ARTIFACT_OS: ubuntu-latest # The default OS for release. + ARTIFACT_PYTHON: '3.11' # The default Python version for release. + +jobs: + build: + outputs: + artifacts-sha256: ${{ steps.compute-hash.outputs.artifacts-sha256 }} + name: Build Python ${{ matrix.python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # It is recommended to pin a Runner version specifically: + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners + os: [ubuntu-latest] + python: ['3.11'] + steps: + + - name: Check out repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + with: + python-version: ${{ matrix.python }} + + # Using the Makefile assumes an activated virtual environment, which doesn't exist + # when running in an Action environment (https://github.com/actions/setup-python/issues/359). + # Instead we create an empty .venv folder so that the Makefile continues to function + # while Python operates within the runner's global environment. It is safe to ignore + # warnings from the Makefile about the missing virtual environment. + - name: Create empty virtual environment for Actions + run: mkdir .venv + - name: Install dependencies + run: make setup + + # Audit all currently installed packages for security vulnerabilities. + - name: Audit installed packages + run: make audit + + # Build the sdist and wheel distribution of the package and docs as a zip file. + # We don't need to check and test the package separately because `make dist` runs + # those targets first and only builds the package if they succeed. + - name: Build the package + run: make dist + env: + GITHUB_TOKEN: ${{ github.token }} + + # Generate the requirements.txt that contains the hash digests of the dependencies and + # generate the SBOM using CyclonDX SBOM generator. + - name: Generate requirements.txt and SBOM + if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON + run: make requirements sbom + + # Remove the old requirements.txt file (which includes _all_ packages) and generate a + # new one for the package and its actual and required dependencies only. + - name: Prune packages and generate required requirements.txt + if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON + run: | + rm requirements.txt + make prune requirements + + # Find the paths to the artifact files that will be included in the release, compute + # the SHA digest for all the release files and encode them using Base64, and export it + # from this job. + - name: Compute package hash + if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON + id: compute-hash + shell: bash + run: | + set -euo pipefail + TARBALL_PATH=$(find dist/ -type f -name "*.tar.gz") + WHEEL_PATH=$(find dist/ -type f -name "*.whl") + GO_ACTION_PARSER=$(find bin/ -type f -name "actionparser") + GO_BASH_PARSER=$(find bin/ -type f -name "bashparser") + REQUIREMENTS_PATH=$(find dist/ -type f -name "*-requirements.txt") + SBOM_PATH=$(find dist/ -type f -name "*-sbom.json") + SBOM_GO_PATH=$(find dist/ -type f -name "*-sbom-go.json") + HTML_DOCS_PATH=$(find dist/ -type f -name "*-docs-html.zip") + BUILD_EPOCH_PATH=$(find dist/ -type f -name "*-build-epoch.txt") + DIGEST=$(sha256sum "$TARBALL_PATH" "$WHEEL_PATH" "$REQUIREMENTS_PATH" "$SBOM_PATH" \ + "$SBOM_GO_PATH" "$GO_ACTION_PARSER" "$GO_BASH_PARSER" "$HTML_DOCS_PATH" "$BUILD_EPOCH_PATH" | base64 -w0) + echo "Digest of artifacts is $DIGEST." + echo "artifacts-sha256=$DIGEST" >> "$GITHUB_OUTPUT" + + # For now only generate artifacts for the specified OS and Python version in env variables. + # Currently reusable workflows do not support setting strategy property from the caller workflow. + - name: Upload the package artifact for debugging and release + if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON + uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 + with: + name: artifact-${{ matrix.os }}-python-${{ matrix.python }} + path: | + dist + bin/actionparser + bin/bashparser + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/_release-notifications.yaml b/.github/workflows/_release-notifications.yaml new file mode 100644 index 000000000..246f6bc2c --- /dev/null +++ b/.github/workflows/_release-notifications.yaml @@ -0,0 +1,50 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# Send a Slack release notification. Instructions to set up Slack to receive +# messages can be found here: https://github.com/slackapi/slack-github-action#setup-2 + +name: Release Notifications +on: + workflow_call: + inputs: + repo_name: + required: true + type: string + release_tag: + required: true + type: string + release_url: + required: true + type: string + secrets: + SLACK_WEBHOOK_URL: + required: true + +# Grant no permissions to this workflow. +permissions: {} + +jobs: + slack: + name: Slack release notification + runs-on: ubuntu-latest + steps: + + - name: Notify via Slack + run: | + curl --header "Content-Type: application/json; charset=UTF-8" --request POST --data "$SLACK_WEBHOOK_MSG" "$SLACK_WEBHOOK_URL" + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_MSG: | + { + "text": "${{ inputs.repo_name }} published a new release ${{ inputs.release_tag }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*${{ inputs.repo_name }}* published a new release <${{ inputs.release_url }}|${{ inputs.release_tag }}>" + } + } + ] + } diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml new file mode 100644 index 000000000..a212720e5 --- /dev/null +++ b/.github/workflows/codeql-analysis.yaml @@ -0,0 +1,71 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# Run CodeQL over the package. For more configuration options see codeql/codeql-config.yaml +# and: https://github.com/github/codeql-action + +name: CodeQL +on: + push: + branches: + - main + - staging + pull_request: + branches: + - main + - staging + # Avoid unnecessary scans of pull requests. + paths: + - '**/*.py' + schedule: + - cron: 20 15 * * 3 +permissions: + contents: read + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + language: [python] + python: ['3.11'] + steps: + + - name: Checkout repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + with: + python-version: ${{ matrix.python }} + + # For more details see the comment in _build.yaml. + - name: Create empty virtual environment for Actions + run: mkdir .venv + - name: Install dependencies + run: make setup + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@c3b6fce4ee2ca25bc1066aa3bf73962fda0e8898 # v2.1.31 + with: + languages: ${{ matrix.language }} + config-file: .github/codeql/codeql-config.yaml + # Override the default behavior so that the action doesn't attempt + # to auto-install Python dependencies + setup-python-dependencies: false + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@c3b6fce4ee2ca25bc1066aa3bf73962fda0e8898 # v2.1.31 diff --git a/.github/workflows/dependabot-automerge.yaml b/.github/workflows/dependabot-automerge.yaml new file mode 100644 index 000000000..045c3cca4 --- /dev/null +++ b/.github/workflows/dependabot-automerge.yaml @@ -0,0 +1,24 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# Automatically merge Dependabot PRs upon approval by leaving +# a comment on Dependabot's pull-request. + +name: Automerge Dependabot PR +on: + pull_request_review: + types: [submitted] + +permissions: + pull-requests: write + +jobs: + comment: + if: ${{ github.event.review.state == 'approved' && github.event.pull_request.user.login == 'dependabot[bot]' }} + runs-on: ubuntu-latest + steps: + - name: Merge Dependabot PR + run: gh pr comment --body "@dependabot squash and merge" "$PR_URL" + env: + PR_URL: ${{ github.event.pull_request.html_url }} + GITHUB_TOKEN: ${{ secrets.DEPENDABOT_AUTOMERGE_TOKEN }} diff --git a/.github/workflows/pr-change-set.yaml b/.github/workflows/pr-change-set.yaml new file mode 100644 index 000000000..cf7e230c2 --- /dev/null +++ b/.github/workflows/pr-change-set.yaml @@ -0,0 +1,23 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# This workflow checks and tests the package code, and it builds all package +# artifacts whenever there were changes to a pull request. + +name: Check change set +on: + pull_request: + branches: + - '*' + types: + - opened + - reopened + - synchronize +permissions: + contents: read + +jobs: + build: + uses: ./.github/workflows/_build.yaml + permissions: + contents: read diff --git a/.github/workflows/pr-conventional-commits.yaml b/.github/workflows/pr-conventional-commits.yaml new file mode 100644 index 000000000..867b75697 --- /dev/null +++ b/.github/workflows/pr-conventional-commits.yaml @@ -0,0 +1,58 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# This workflow lints the PR's title and commits. It uses the commitizen +# package (https://github.com/commitizen-tools/commitizen) and its `cz` +# tool to check the title of the PR and all commit messages of the branch +# which triggers this Action. + +name: Check conventional commits +on: + pull_request: + branches: + - '*' + types: + - opened + - reopened + - edited + - synchronize +permissions: + contents: read + +jobs: + conventional-commits: + runs-on: ubuntu-latest + steps: + + - name: Check out repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + with: + python-version: '3.11' + + # Install Commitizen without using the package's Makefile: that's much faster than + # creating a venv and installing heaps of dependencies that aren't required for this job. + - name: Set up Commitizen + run: | + pip install --upgrade pip wheel + pip install 'commitizen ==2.37.1' + + # Run Commitizen to check the title of the PR which triggered this workflow, and check + # all commit messages of the PR's branch. If any of the checks fails then this job fails. + - name: Check PR title + run: echo "$PR_TITLE" | cz check + env: + PR_TITLE: ${{ github.event.pull_request.title }} + - name: Check PR commit messages + run: | + git remote add other "$PR_HEAD_REPO_CLONE_URL" + git fetch other + cz check --rev-range "origin/$PR_BASE_REF..other/$PR_HEAD_REF" + env: + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + PR_HEAD_REPO_CLONE_URL: ${{ github.event.pull_request.head.repo.clone_url }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..761051fc1 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,214 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# We run checks on pushing to the specified branches. +# Pushing to main also triggers a release. + +name: Check and Release +on: + workflow_dispatch: + # push: + # branches: + # - main + # - staging +permissions: + contents: read +env: + ARTIFACT_NAME: artifact-ubuntu-latest-python-3.11 + # This is the username and email for the user who commits and pushes the release + # commit. In an organisation that should be a dedicated devops account. + USER_NAME: behnazh-w + USER_EMAIL: behnazh-w@users.noreply.github.com + +jobs: + check: + if: ${{ !startsWith(github.event.commits[0].message, 'bump:') }} + uses: ./.github/workflows/_build.yaml + permissions: + contents: read + + # On pushes to the 'main' branch create a new release by bumping the version + # and generating a change log. That's the new bump commit and associated tag. + bump: + needs: check + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + + - name: Check out repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + fetch-depth: 0 + token: ${{ secrets.REPO_ACCESS_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + with: + python-version: '3.11' + + - name: Set up Commitizen + run: | + pip install --upgrade pip wheel + pip install 'commitizen ==2.37.1' + + - name: Set up user + run: | + git config --global user.name "$USER_NAME" + git config --global user.email "$USER_EMAIL" + git config --list --global # For debug purposes. + + - name: Create changelog and bump + run: cz bump --changelog --yes + + - name: Push the release + run: | + git push + git push --tags + + # When triggered by the version bump commit, build the package and publish the release artifacts. + build: + if: github.ref == 'refs/heads/main' && startsWith(github.event.commits[0].message, 'bump:') + uses: ./.github/workflows/_build.yaml + permissions: + contents: read + + # Create a new Release on Github from the verified build artifacts, and optionally + # publish the artifacts to a PyPI server. + release: + needs: [build] + name: Release + outputs: + release-tag: ${{ steps.upload-assets.outputs.release-tag }} + release-url: ${{ steps.upload-assets.outputs.release-url }} + runs-on: ubuntu-latest + permissions: + contents: write # To publish release notes. + steps: + + - name: Check out repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + fetch-depth: 0 + + - name: Download artifact + uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # v3.0.1 + with: + name: ${{ env.ARTIFACT_NAME }} + path: dist + + # Verify hashes by first computing hashes for the artifacts and then comparing them + # against the hashes computed by the build job. + - name: Verify the artifact hash + env: + ARTIFACT_HASH: ${{ needs.build.outputs.artifacts-sha256 }} + run: | + set -euo pipefail + echo "Hash of package should be $ARTIFACT_HASH." + echo "$ARTIFACT_HASH" | base64 --decode | sha256sum --strict --check --status || exit 1 + + # Create the Release Notes using commitizen. + - name: Set up Python + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + with: + python-version: '3.11' + + - name: Set up Commitizen + run: | + pip install --upgrade pip wheel + pip install 'commitizen ==2.37.1' + + - name: Create Release Notes + run: cz changelog --dry-run "$(cz version --project)" > RELEASE_NOTES.md + + # Create the release including the artifacts and the SLSA L3 provenance. + - name: Upload assets + id: upload-assets + env: + GH_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + run: | + TAG=$(git describe --tags --abbrev=0) + gh release create "$TAG" dist/* --title "$TAG" --notes-file RELEASE_NOTES.md + echo "release-tag=$TAG" >> "$GITHUB_OUTPUT" + echo "release-url=$(gh release view """$TAG""" --json url --jq .url)" >> "$GITHUB_OUTPUT" + + # Uncomment the following steps to publish to a PyPI server. + # At the moment PyPI does not provide a mechanism to publish + # the provenance. So, users have to download the provenance from + # the release page of the GitHub repository to verify the artifact. + # Install Twine without using the package's Makefile to avoid + # installing unnecessary dependencies, which is slow. + # - name: Set up Twine + # run: | + # pip install --upgrade pip wheel + # pip install 'twine ==4.0.1' + + # Pass the username, password, and PYPI repository URL via env variables. + # Read the password from GitHub secrets or via other trusted mechanisms. + # Do not hardcode the password in the workflow. + # - name: Publish to PyPI server + # run: twine upload --verbose dist/*.tar.gz dist/*.whl + # env: + # TWINE_USERNAME= + # TWINE_PASSWORD= + # TWINE_REPOSITORY_URL= + + # Generate the build provenance. The generator should be referenced with a semantic version. + # The build will fail if we reference it using the commit SHA. To avoid using a pre-built + # provenance generator which depends on an external service Rekor (https://github.com/sigstore/rekor) + # we build this generator from source for now. For more information see this discussion: + # https://github.com/slsa-framework/slsa-github-generator/issues/942 + provenance: + needs: [build, release] + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.2.2 + with: + base64-subjects: ${{ needs.build.outputs.artifacts-sha256 }} + compile-generator: true # Build the generator from source. + # Set private-repository to true for private repositories. Note that the repository name is + # uploaded as part of the transparency log entry on the public Rekor instance (rekor.sigstore.dev). + private-repository: false + permissions: + actions: read # To read the workflow path. + id-token: write # To sign the provenance. + contents: write # To add assets to a release. + + # Publish the SLSA provenance as the GitHub release asset. + publish_provenance: + needs: [release, provenance] + name: Publish provenance + runs-on: ubuntu-latest + permissions: + contents: write # To publish release notes. + steps: + + - name: Check out repository + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2 + with: + fetch-depth: 0 + + - name: Download provenance + uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # v3.0.1 + with: + name: ${{ needs.provenance.outputs.provenance-name }} + + - name: Upload provenance + run: gh release upload ${{ needs.release.outputs.release-tag }} ${{ needs.provenance.outputs.provenance-name }} + env: + GH_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + + # Send out release notifications after the Release was published on GitHub. + # Uncomment the `if` to disable sending release notifications. + notifications: + # if: ${{ false }} + needs: [release] + name: Send Release notifications + uses: ./.github/workflows/_release-notifications.yaml + permissions: + contents: read + with: + repo_name: ${{ github.event.repository.name }} + release_tag: ${{ needs.release.outputs.release-tag }} + release_url: ${{ needs.release.outputs.release-url }} + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/scorecards-analysis.yaml b/.github/workflows/scorecards-analysis.yaml new file mode 100644 index 000000000..d9ccafb34 --- /dev/null +++ b/.github/workflows/scorecards-analysis.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# Run Scorecard for this repository to further check and harden software and process. + +name: Scorecards supply-chain security +on: + # Only the default branch is supported. + branch_protection_rule: + schedule: + - cron: 27 20 * * 1 + push: + branches: [main] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecards analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + actions: read + contents: read + # Needed to access OIDC token. + id-token: write + steps: + + - name: Check out repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6 + with: + results_file: results.sarif + results_format: sarif + # Read-only PAT token. To create it, + # follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. + repo_token: ${{ secrets.SCORECARD_READ_TOKEN }} + # Publish the results to enable scorecard badges. For more details, see + # https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories, `publish_results` will automatically be set to `false`, + # regardless of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). + - name: Upload artifact + uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 + with: + name: SARIF file + path: results.sarif + + # Upload the results to GitHub's code scanning dashboard. + - name: Upload to code-scanning + uses: github/codeql-action/upload-sarif@c3b6fce4ee2ca25bc1066aa3bf73962fda0e8898 # v2.1.31 + with: + sarif_file: results.sarif diff --git a/.gitignore b/.gitignore index b2a0e56a7..db5f9440c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,142 @@ # Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +include/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Local venv +bin/ +pyvenv.cfg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +#Macaron __pycache__ .pyc .idea @@ -17,6 +153,10 @@ git_repos .pytest_cache .mvn mvnw +mvnw.cmd +mvnwDebug +mvnwDebug.cmd +maven-wrapper-distribution-* build_log macaron.db gradlew @@ -25,11 +165,11 @@ gradlew reports output cdx_debug.json -osint_debug.json golang/internal/filewriter/mock_dir/result.json tests/config/defaults.ini tests/defaults.ini tests/slsa_analyzer/checks/mock_repos/** tests/slsa_analyzer/ci_service/mock_repos/** -docs/build +docs/_build bin/ +requirements.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..35e34c6c3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,216 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +default_install_hook_types: [pre-commit, commit-msg, pre-push] +default_stages: [commit] +minimum_pre_commit_version: 2.18.0 + +repos: + +# These meta hooks check the pre-commit configuration itself. +- repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + +# Commitizen enforces semantic and conventional commit messages. +- repo: https://github.com/commitizen-tools/commitizen + rev: v2.37.1 + hooks: + - id: commitizen + name: Check conventional commit message + stages: [commit-msg] + +# Sort imports. +- repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + name: Sort import statements + args: [--settings-path, pyproject.toml] + +# Add Black code formatters. +- repo: https://github.com/ambv/black + rev: 22.10.0 + hooks: + - id: black + name: Format code + args: [--config, pyproject.toml] +- repo: https://github.com/asottile/blacken-docs + rev: v1.12.1 + hooks: + - id: blacken-docs + name: Format code in docstrings + args: [--line-length, '120'] + additional_dependencies: [black==22.10.0] + +# Upgrade and rewrite Python idioms. +- repo: https://github.com/asottile/pyupgrade + rev: v3.2.3 + hooks: + - id: pyupgrade + name: Upgrade code idioms + files: ^src/macaron/|^tests/ + args: [--py39-plus] + +# Similar to pylint, with a few more/different checks. For more available +# extensions: https://github.com/DmytroLitvinov/awesome-flake8-extensions +- repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + name: Check flake8 issues + files: ^src/macaron/|^tests/ + types: [text, python] + additional_dependencies: [flake8-bugbear==22.10.27, flake8-builtins==2.0.1, flake8-comprehensions==3.10.1, flake8-docstrings==1.6.0, flake8-mutable==1.2.0, flake8-noqa==1.3.0, flake8-pytest-style==1.6.0, flake8-rst-docstrings==0.3.0, pep8-naming==0.13.2] + args: [--config, .flake8] + +# Run Pylint from the local repo to make sure venv packages +# specified in pyproject.toml are available. +- repo: local + hooks: + - id: pylint + name: Check pylint issues + entry: pylint + language: python + files: ^src/macaron/|^tests/ + types: [text, python] + args: [--rcfile, pyproject.toml] + +# Type-check all Python code. +- repo: local + hooks: + - id: mypy + name: Check typing annotations + entry: mypy + language: python + files: ^src/macaron/|^tests/ + types: [text, python] + args: [--show-traceback, --config-file, pyproject.toml] + +# Check for potential security issues. +- repo: https://github.com/PyCQA/bandit + rev: 1.7.4 + hooks: + - id: bandit + name: Check for security issues + args: [--configfile, pyproject.toml] + files: ^src/macaron/|^tests/ + types: [text, python] + additional_dependencies: ['bandit[toml]'] + +# Enable a whole bunch of useful helper hooks, too. +# See https://pre-commit.com/hooks.html for more hooks. +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-ast + - id: check-case-conflict + - id: check-merge-conflict + - id: debug-statements + - id: end-of-file-fixer + stages: [commit] + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + stages: [commit] + - id: detect-private-key + - id: detect-aws-credentials + args: [--allow-missing-credentials] + - id: check-yaml + args: [--allow-multiple-documents] + - id: check-toml +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.9.0 + hooks: + - id: python-check-blanket-noqa + # Disabling blanket-type-ignore for now until type stubs are added. + # - id: python-check-blanket-type-ignore + - id: python-check-mock-methods + - id: python-use-type-annotations + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + - id: text-unicode-replacement-char + +# Check the reStructured Text files that make up +# this package's documentation. +# Commenting this out because https://github.com/Lucas-C/pre-commit-hooks-markup/issues/13 +# - repo: https://github.com/Lucas-C/pre-commit-hooks-markup +# rev: v1.0.1 +# hooks: +# - id: rst-linter + +# Check and prettify the configuration files. +- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.4.0 + hooks: + - id: pretty-format-ini + args: [--autofix] + - id: pretty-format-yaml + args: [--autofix] + # Commenting this out because https://github.com/pappasam/toml-sort/issues/11 + # - id: pretty-format-toml + # args: [--autofix] + +# On push to the remote, run the unit tests. +- repo: local + hooks: + - id: pytest + name: Run unit tests + entry: python -W always::DeprecationWarning -m pytest -c pyproject.toml --cov-config pyproject.toml + language: system + verbose: true + always_run: true + pass_filenames: false + stages: [push] + +# Checks the copyright header for .js, .py, and .java, etc. files. +- repo: local + hooks: + - id: copyright-checker + name: Copyright checker + entry: scripts/dev_scripts/copyright-checker.sh + language: system + always_run: true + pass_filenames: false + +# A linter for Golang +- repo: https://github.com/golangci/golangci-lint + rev: v1.50.1 + hooks: + - id: golangci-lint + +# Other pre-commit hooks for golang: +# *-mod hooks run on the MODULE of each staged Go files. +# *-repo-mod hooks run on each MODULE in the repository. This allows us to add multiple go modules to this repo. +# *-repo hooks only run in the REPOSITORY root folder. +# These hooks automatically run on staged Go files, go.mod and go.sum. +# Other staged files shouldn't trigger these hooks. +# Documentation: https://github.com/TekWizely/pre-commit-golang/blob/v1.0.0-rc.1/README.md. +- repo: https://github.com/tekwizely/pre-commit-golang + rev: v1.0.0-rc.1 + hooks: + - id: go-build-mod + - id: go-build-repo-mod + # + # Go Mod Tidy + # + - id: go-mod-tidy + - id: go-mod-tidy-repo + # + # Go Test + # + - id: go-test-mod + - id: go-test-repo-mod + # + # Go Vet + # + - id: go-vet-mod + - id: go-vet-repo-mod + # + # Go formatters + # + - id: go-fmt + - id: go-fmt-repo diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2434400a..b202fdf35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,4 +76,4 @@ can be accepted. Follow the [Golden Rule](https://en.wikipedia.org/wiki/Golden_Rule). If you'd like more specific guidelines, see the [Contributor Covenant Code of Conduct][COC]. [OCA]: https://oca.opensource.oracle.com -[COC]: https://www.contributor-covenant.org/version/1/4/code-of-conduct/ \ No newline at end of file +[COC]: https://www.contributor-covenant.org/version/1/4/code-of-conduct/ diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 24a89357d..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. - -recursive-include src/macaron * -recursive-exclude src/macaron/**/__pycache__ * - -exclude .gitignore .pre-commit-config.yaml .flake8 -exclude MANIFEST.in -exclude pyproject.toml -recursive-exclude docs * -recursive-exclude tests * diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..f0d1db7e8 --- /dev/null +++ b/Makefile @@ -0,0 +1,267 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# Use bash as the shell when executing a rule's recipe. For more details: +# https://www.gnu.org/software/make/manual/html_node/Choosing-the-Shell.html +SHELL := bash + +# Set the package's name, version, and path for use throughout the Makefile. +PACKAGE_NAME := macaron +PACKAGE_VERSION := $(shell python -c $$'try: import $(PACKAGE_NAME); print($(PACKAGE_NAME).__version__);\nexcept: print("unknown");') +MACARON_PATH := $(shell pwd) +PYTHON ?= python3.11 + +# This variable contains the first goal that matches any of the listed goals +# here, else it contains an empty string. The net effect is to filter out +# whether this current run of `make` requires a Python virtual environment +# by checking if any of the given goals requires a virtual environment (all +# except the 'venv' and the various 'clean' and 'nuke' goals do). Note that +# checking for 'upgrade' and 'check' goals includes all of their variations. +NEED_VENV := $(or \ + $(findstring all,$(MAKECMDGOALS)), \ + $(findstring setup,$(MAKECMDGOALS)), \ + $(findstring upgrade,$(MAKECMDGOALS)), \ + $(findstring sbom,$(MAKECMDGOALS)), \ + $(findstring requirements,$(MAKECMDGOALS)), \ + $(findstring audit,$(MAKECMDGOALS)), \ + $(findstring check,$(MAKECMDGOALS)), \ + $(findstring test,$(MAKECMDGOALS)), \ + $(findstring test-integration,$(MAKECMDGOALS)), \ + $(findstring dist,$(MAKECMDGOALS)), \ + $(findstring docs,$(MAKECMDGOALS)), \ + $(findstring prune,$(MAKECMDGOALS)), \ +) +ifeq ($(NEED_VENV),) + # None of the current goals requires a virtual environment. +else + ifeq ($(origin VIRTUAL_ENV),undefined) + $(warning No Python virtual environment found, proceeding anyway) + else + ifeq ($(wildcard .venv/upgraded-on),) + $(warning Python virtual environment not yet set up, proceeding anyway) + endif + endif +endif + +# If the project configuration file has been updated (package deps or +# otherwise) then warn the user and suggest resolving the conflict. +ifeq ($(shell test pyproject.toml -nt .venv/upgraded-on; echo $$?),0) + $(warning pyproject.toml was updated, consider `make upgrade` if your packages have changed) + $(warning If this is not correct then run `make upgrade-quiet`) +endif + +# The SOURCE_DATE_EPOCH environment variable allows the `flit` tool to +# reproducibly build packages: https://flit.pypa.io/en/latest/reproducible.html +# If that variable doesn't exist, then set it here to the current epoch. +ifeq ($(origin SOURCE_DATE_EPOCH),undefined) + SOURCE_DATE_EPOCH := $(shell date +%s) +endif + +# Check, test, and build artifacts for this package. +.PHONY: all +all: check test dist docs + +# Create a virtual environment, either for Python or using +# the Python interpreter specified in the PYTHON environment variable. Also +# create an empty pip.conf file to ensure that `pip config` modifies this +# venv only, unless told otherwise. +.PHONY: venv +venv: + if [ ! -z "${VIRTUAL_ENV}" ]; then \ + echo "Found an activated Python virtual environment, exiting" && exit 1; \ + fi + if [ -d .venv/ ]; then \ + echo "Found an inactive Python virtual environment, please activate or nuke it" && exit 1; \ + fi + echo "Creating virtual environment in .venv/ for ${PYTHON}"; \ + ${PYTHON} -m venv --upgrade-deps --prompt . .venv; \ + touch .venv/pip.conf + +# Set up a newly created virtual environment. Note: pre-commit uses the +# venv's Python interpreter, so if you've created multiple venvs then +# pre-commit's git hooks run against the most recently set up venv. +# The _build.yaml GitHub Actions workflow expects dist directory to exist. +# So we create the dist dir if it doesn't exist in the setup target. +# See https://packaging.python.org/en/latest/tutorials/packaging-projects/#generating-distribution-archives. +# We also install SLSA verifier, mvnw, cyclonedx-go, and compile the Go modules. +.PHONY: setup +setup: force-upgrade setup-go + pre-commit install + mkdir -p dist + git clone --depth 1 https://github.com/slsa-framework/slsa-verifier.git -b v2.0.1 + cd slsa-verifier/cli/slsa-verifier && go build -o $(MACARON_PATH)/bin/ + cd $(MACARON_PATH) && rm -rf slsa-verifier + go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@v1.3.0 + echo "GOPATH=$$GOPATH" + ls $$HOME/go/bin + cd resources \ + && wget https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper-distribution/3.1.1/maven-wrapper-distribution-3.1.1-bin.zip \ + && unzip -o maven-wrapper-distribution-3.1.1-bin.zip \ + && rm -r maven-wrapper-distribution-3.1.1-bin.zip \ + && echo -e "distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip\nwrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar" > .mvn/wrapper/maven-wrapper.properties \ + && cd $(MACARON_PATH) +setup-go: + go build -o $(MACARON_PATH)/bin/ $(MACARON_PATH)/golang/cmd/... + +# Install or upgrade an existing virtual environment based on the +# package dependencies declared in pyproject.toml and go.mod. +.PHONY: upgrade force-upgrade upgrade-go +upgrade: .venv/upgraded-on upgrade-go +.venv/upgraded-on: pyproject.toml + python -m pip install --upgrade pip + python -m pip install --upgrade wheel + python -m pip install --upgrade --upgrade-strategy eager --editable .[actions,dev,docs,hooks,test] + $(MAKE) upgrade-quiet +force-upgrade: upgrade-go + rm -f .venv/upgraded-on + $(MAKE) upgrade +upgrade-quiet: + echo "Automatically generated by Python Package Makefile on $$(date '+%Y-%m-%d %H:%M:%S %z')." > .venv/upgraded-on +upgrade-go: + go get -u ./... + go mod tidy + +# Generate a Software Bill of Materials (SBOM). +.PHONY: sbom +sbom: requirements + cyclonedx-bom --force --requirements --format json --output dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-sbom.json + $$HOME/go/bin/cyclonedx-gomod mod -json -output dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-sbom-go.json $(MACARON_PATH) + +# Generate a requirements.txt file containing version and integrity hashes for all +# packages currently installed in the virtual environment. There's no easy way to +# do this, see also: https://github.com/pypa/pip/issues/4732 +# +# If using a private package index, make sure that it implements the JSON API: +# https://warehouse.pypa.io/api-reference/json.html +# +# We also want to make sure that this package itself is added to the requirements.txt +# file, and if possible even with proper hashes. +.PHONY: requirements +requirements: requirements.txt +requirements.txt: pyproject.toml + echo -n "" > requirements.txt + for pkg in $$(python -m pip freeze --local --disable-pip-version-check --exclude-editable); do \ + pkg=$${pkg//[$$'\r\n']}; \ + echo -n $$pkg >> requirements.txt; \ + echo "Fetching package metadata for requirement '$$pkg'"; \ + [[ $$pkg =~ (.*)==(.*) ]] && curl -s https://pypi.org/pypi/$${BASH_REMATCH[1]}/$${BASH_REMATCH[2]}/json | python -c "import json, sys; print(''.join(f''' \\\\\n --hash=sha256:{pkg['digests']['sha256']}''' for pkg in json.load(sys.stdin)['urls']));" >> requirements.txt; \ + done + echo -e -n "$(PACKAGE_NAME)==$(PACKAGE_VERSION)" >> requirements.txt + if [ -f dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz ]; then \ + echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz | grep '^\-\-hash')" >> requirements.txt; \ + fi + if [ -f dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl ]; then \ + echo -e -n " \\\\\n $$(python -m pip hash --algorithm sha256 dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl | grep '^\-\-hash')" >> requirements.txt; \ + fi + echo "" >> requirements.txt + cp requirements.txt dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-requirements.txt + +# Audit the currently installed packages. Skip packages that are installed in +# editable mode (like the one in development here) because they may not have +# a PyPI entry; also print out CVE description and potential fixes if audit +# found an issue. +# TODO: do not ignore GHSA-hcpj-qp55-gfph once the patch is out. +# See: https://github.com/gitpython-developers/GitPython/issues/1515. +.PHONY: audit +audit: + if ! $$(python -c "import pip_audit" &> /dev/null); then \ + echo "No package pip_audit installed, upgrade your environment!" && exit 1; \ + fi; + python -m pip_audit --skip-editable --desc on --fix --dry-run --ignore-vuln GHSA-hcpj-qp55-gfph + +# Run some or all checks over the package code base. +.PHONY: check check-code check-bandit check-flake8 check-lint check-mypy check-go +check-code: check-bandit check-flake8 check-lint check-mypy check-go +check-bandit: + pre-commit run bandit --all-files +check-flake8: + pre-commit run flake8 --all-files +check-lint: + pre-commit run pylint --all-files +check-mypy: + pre-commit run mypy --all-files +check-go: + pre-commit run golangci-lint --all-files + pre-commit run go-build-mod --all-files + pre-commit run go-build-repo-mod --all-files + pre-commit run go-mod-tidy --all-files + pre-commit run go-mod-tidy-repo --all-files + pre-commit run go-test-mod --all-files + pre-commit run go-test-repo-mod --all-files + pre-commit run go-vet-mod --all-files + pre-commit run go-vet-repo-mod --all-files + pre-commit run go-fmt --all-files + pre-commit run go-fmt-repo --all-files +check: + pre-commit run --all-files + + +# Run all unit tests. The --files option avoids stashing but passes files; however, +# the hook setup itself does not pass files to pytest (see .pre-commit-config.yaml). +.PHONY: test +test: test-go + pre-commit run pytest --hook-stage push --files tests/ +test-go: + go test ./golang/... + +# Run the integration tests. +.PHONY: integration-test +integration-test: + scripts/dev_scripts/integration_tests.sh $(MACARON_PATH) $$HOME + +# Build a source distribution package and a binary wheel distribution artifact. +# When building these artifacts, we need the environment variable SOURCE_DATE_EPOCH +# set to the build date/epoch. For more details, see: https://flit.pypa.io/en/latest/reproducible.html +.PHONY: dist +dist: dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt +dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl: check test integration-test + flit build --setup-py --format wheel +dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz: check test integration-test + flit build --setup-py --format sdist +dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip: docs + python -m zipfile -c dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip docs/_build/html +dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt: + echo $(SOURCE_DATE_EPOCH) > dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt + +# Build the HTML documentation from the package's source. +.PHONY: docs +docs: docs/_build/html/index.html +docs/_build/html/index.html: + $(MAKE) -C docs/ html + +# Prune the packages currently installed in the virtual environment down to the required +# packages only. Pruning works in a roundabout way, where we first generate the wheels for +# all installed packages into the build/wheelhouse/ folder. Next we wipe all packages and +# then reinstall them from the wheels while disabling the PyPI index server. Thus we ensure +# that the same package versions are reinstalled. Use with care! +.PHONY: prune +prune: + mkdir -p build/ + python -m pip freeze --local --disable-pip-version-check --exclude-editable > build/prune-requirements.txt + python -m pip wheel --wheel-dir build/wheelhouse/ --requirement build/prune-requirements.txt + python -m pip wheel --wheel-dir build/wheelhouse/ . + python -m pip uninstall --yes --requirement build/prune-requirements.txt + python -m pip install --no-index --find-links=build/wheelhouse/ --editable . + rm -fr build/ + +# Clean test caches and remove build artifacts. +.PHONY: dist-clean bin-clean clean +dist-clean: + rm -fr dist/* + rm -f requirements.txt +bin-clean: + rm -fr bin/* +clean: dist-clean bin-clean + rm -fr .coverage .hypothesis/ .mypy_cache/ .pytest_cache/ + rm -fr docs/_build/ + +# Remove code caches, or the entire virtual environment if it is deactivated.. +.PHONY: nuke-caches nuke +nuke-caches: clean + find src/ -type d -name __pycache__ -exec rm -fr {} + + find tests/ -type d -name __pycache__ -exec rm -fr {} + +nuke: nuke-caches + if [ ! -z "${VIRTUAL_ENV}" ]; then \ + echo "Please deactivate the virtual environment first!" && exit 1; \ + fi + rm -fr .venv/ diff --git a/README.md b/README.md index d9fcaefdb..92bc869e0 100644 --- a/README.md +++ b/README.md @@ -18,47 +18,18 @@ Macaron uses [SLSA requirements specifications v0.1](https://slsa.dev/spec/v0.1/ ## Getting started **Prerequisites** -- Python 3.10.5 +- Python 3.11 - Go 1.18 - JDK 11 **Prepare the environment** -```bash -python -m venv .venv -. .venv/bin/activate -``` - Clone the project and install Macaron. ```bash -python -m pip install --editable . -``` - -Build Macaron's Go modules: - -```bash -go build -o ./bin/ ./golang/cmd/... -``` - -Download and build [slsa-verifer](https://github.com/slsa-framework/slsa-verifier): - -```bash -MACARON_PATH=$(pwd) -git clone --depth 1 https://github.com/slsa-framework/slsa-verifier.git -b -cd slsa-verifier/cli/slsa-verifier && go build -o $MACARON_PATH/bin/ -cd $MACARON_PATH && rm -rf slsa-verifier -``` - -Download and install Maven wrapper: - -```bash -cd resources \ - && wget https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper-distribution/3.1.1/maven-wrapper-distribution-3.1.1-bin.zip \ - && unzip maven-wrapper-distribution-3.1.1-bin.zip \ - && rm -r maven-wrapper-distribution-3.1.1-bin.zip \ - && echo -e "distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip\nwrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar" > .mvn/wrapper/maven-wrapper.properties \ - && cd .. +make venv +. .venv/bin/activate +make setup ``` ## Running Macaron @@ -163,10 +134,98 @@ python -m macaron -po verify -pr **Note.** The policy engine is under active development and will support more complex policies soon. Stay tuned. - ## How to Contribute -We welcome contributions! See our [contribution guidelines](./CONTRIBUTING.md). +We welcome contributions! See our [general contribution guidelines](./CONTRIBUTING.md). + +To contribute to Macaron, first create a [virtual environment](https://docs.python.org/3/tutorial/venv.html) by either using the [Makefile](https://www.gnu.org/software/make/manual/make.html#toc-An-Introduction-to-Makefiles): + +```bash +make venv # Create a new virtual environment in .venv folder using Python 3.11. +``` + +or for a specific version of Python: + +```bash +PYTHON=python3.11 make venv # Same virtual environment for a different Python version. +``` + +Activate the virtual environment: + +```bash +. .venv/bin/activate +``` + +Finally, set up Macaron with all of its extras and initialize the local git hooks: + +```bash +make setup +``` + +With that in place, you’re ready to build and contribute to Macaron! + +### Updating dependent packages + +It’s likely that during development you’ll add or update dependent packages in the `pyproject.toml` file, which requires an update to the virtual environment: + +```bash +make upgrade +``` + +### Git hooks + +Using the pre-commit tool and its `.pre-commit-config.yaml` configuration, the following git hooks are active in this repository: + +- When committing code, a number of [pre-commit hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_committing_workflow_hooks) ensure that your code is formatted according to [PEP 8](https://www.python.org/dev/peps/pep-0008/) using the [`black`](https://github.com/psf/black) tool, and they’ll invoke [`flake8`](https://github.com/PyCQA/flake8) (and various plugins), [`pylint`](https://github.com/PyCQA/pylint) and [`mypy`](https://github.com/python/mypy) to check for lint and correct types. There are more checks, but those two are the important ones. You can adjust the settings for these tools in the `pyproject.toml` or `.flake8` configuration files. +- The [commit message hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_committing_workflow_hooks) enforces [conventional commit messages](https://www.conventionalcommits.org/) and that, in turn, enables a _semantic release_ of this package on the Github side: upon merging changes into the `main` branch, the [release action](https://github.com/github.com/oracle-samples/blob/main/.github/workflows/release.yaml) uses the [Commitizen tool](https://commitizen-tools.github.io/commitizen/) to produce a [changelog](https://en.wikipedia.org/wiki/Changelog) and it computes the next version of this package and publishes a release — all based on the commit messages of a release. +- Using a [pre-push hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_other_client_hooks) this package is also set up to run [`pytest`](https://github.com/pytest-dev/pytest); in addition, the [`coverage`](https://github.com/nedbat/coveragepy) plugin makes sure that _all_ of your package’s code is covered by tests and [Hypothesis](https://hypothesis.works/) is already installed to help with generating test payloads. + +You can also run these hooks manually, which comes in very handy during daily development tasks. For example + +```bash +make check-code +``` + +runs all the code checks (i.e. `bandit`, `flake8`, `pylint` and `mypy`), whereas + +```bash +make check +``` + +runs _all_ installed git hooks over your code. For more control over the code checks, the Makefile also implements the `check-bandit`, `check-flake8`, `check-lint`, `check-mypy`, and `check-go` goals. + +### Testing + +As mentioned above, this repository is set up to use [pytest](https://pytest.org/) either standalone or as a pre-push git hook. Tests are stored in the `tests/` folder, and you can run them manually like so: +```bash +make test +``` + +which runs all tests in both your local Python virtual environment. For more options, see the [pytest command-line flags](https://docs.pytest.org/en/6.2.x/reference.html#command-line-flags). Also note that pytest includes [doctest](https://docs.python.org/3/library/doctest.html), which means that module and function [docstrings](https://www.python.org/dev/peps/pep-0257/#what-is-a-docstring) may contain test code that executes as part of the unit tests. + +Test code coverage is already tracked using [coverage](https://github.com/nedbat/coveragepy) and the [pytest-cov](https://github.com/pytest-dev/pytest-cov) plugin for pytest, and it measures how much code in the `src/macaron/` folder is covered by tests. + +Hypothesis is a package that implements [property based testing](https://en.wikipedia.org/wiki/QuickCheck) and that provides payload generation for your tests based on strategy descriptions ([more](https://hypothesis.works/#what-is-hypothesis)). Using its [pytest plugin](https://hypothesis.readthedocs.io/en/latest/details.html#the-hypothesis-pytest-plugin) Hypothesis is ready to be used for this package. + +To run integration tests run: + +```bash +make integration-test +``` + +### Generating documentation + +As mentioned above, all package code should make use of [Python docstrings](https://www.python.org/dev/peps/pep-0257/) in [reStructured text format](https://www.python.org/dev/peps/pep-0287/). Using these docstrings and the documentation template in the `docs/source/` folder, you can then generate proper documentation in different formats using the [Sphinx](https://github.com/sphinx-doc/sphinx/) tool: + +```bash +make docs +``` + +This example generates documentation in HTML, which can then be found here: + +```bash +open docs/_build/html/index.html +``` ## Security issue reports diff --git a/build_spec.yaml b/build_spec.yaml index 5da2f3e08..7a5ffe9c4 100644 --- a/build_spec.yaml +++ b/build_spec.yaml @@ -1,4 +1,5 @@ -# Copyright (c) [year,] year, Oracle and/or its affiliates. +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. version: 0.1 component: build @@ -6,11 +7,11 @@ timeoutInSeconds: 1000 shell: bash steps: - - type: Command - name: "compress the repo" - command: | - tar -cvzf ${OCI_WORKSPACE_DIR}/repo.tgz ./ +- type: Command + name: compress the repo + command: | + tar -cvzf ${OCI_WORKSPACE_DIR}/repo.tgz ./ outputArtifacts: - - name: artifact - type: BINARY - location: ${OCI_WORKSPACE_DIR}/repo.tgz +- name: artifact + type: BINARY + location: ${OCI_WORKSPACE_DIR}/repo.tgz diff --git a/docs/Makefile b/docs/Makefile index 14f78a709..98c1dd00a 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -9,7 +9,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source -BUILDDIR = build +BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: diff --git a/docs/source/conf.py b/docs/source/conf.py index 2ffc0cb89..019665efd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,3 +58,13 @@ "includehidden": False, } html_static_path = ["_static"] + +# We add the docstrings for class constructors in the `__init__` methods. +def skip(app, what, name, obj, would_skip, options): + if name == "__init__": + return False + return would_skip + + +def setup(app): + app.connect("autodoc-skip-member", skip) diff --git a/go.mod b/go.mod index 210c2124d..9f807cd7b 100644 --- a/go.mod +++ b/go.mod @@ -6,19 +6,19 @@ module github.com/oracle-samples/macaron go 1.18 require ( - github.com/rhysd/actionlint v1.6.21 - mvdan.cc/sh/v3 v3.5.1 + github.com/rhysd/actionlint v1.6.22 + mvdan.cc/sh/v3 v3.6.0 ) require ( github.com/fatih/color v1.13.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/rivo/uniseg v0.2.0 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/rivo/uniseg v0.4.3 // indirect github.com/robfig/cron v1.2.0 // indirect - golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect - golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.3.0 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f6018a247..29dff411a 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,9 @@ -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= -github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -21,40 +11,28 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/rhysd/actionlint v1.6.21 h1:OSCP03XnvWSRAhUmA5onpgyGG+3NVoQTCu4UX0Rc2dY= -github.com/rhysd/actionlint v1.6.21/go.mod h1:gIKOdxtV40mBOcD0ZR8EBa8NqjEXToAZioroS3oedMg= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/rhysd/actionlint v1.6.22 h1:cAEf2PGNwJXhdcTVF2xS/0ORqWS+ueUHwjQYsqFsGSk= +github.com/rhysd/actionlint v1.6.22/go.mod h1:gIKOdxtV40mBOcD0ZR8EBa8NqjEXToAZioroS3oedMg= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= +github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/yuin/goldmark v1.4.12/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 h1:Sx/u41w+OwrInGdEckYmEuU5gHoGSL4QbDz3S9s6j4U= -golang.org/x/sys v0.0.0-20220818161305-2296e01440c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -mvdan.cc/editorconfig v0.2.0/go.mod h1:lvnnD3BNdBYkhq+B4uBuFFKatfp02eB6HixDvEz91C0= -mvdan.cc/sh/v3 v3.5.1 h1:hmP3UOw4f+EYexsJjFxvU38+kn+V/s2CclXHanIBkmQ= -mvdan.cc/sh/v3 v3.5.1/go.mod h1:1JcoyAKm1lZw/2bZje/iYKWicU/KMd0rsyJeKHnsK4E= +mvdan.cc/sh/v3 v3.6.0 h1:gtva4EXJ0dFNvl5bHjcUEvws+KRcDslT8VKheTYkbGU= +mvdan.cc/sh/v3 v3.6.0/go.mod h1:U4mhtBLZ32iWhif5/lD+ygy1zrgaQhUu+XFy7C8+TTA= diff --git a/golang/internal/actionparser/resources/invalid.yaml b/golang/internal/actionparser/resources/invalid.yaml index 3d9ac1c24..9f0c2a6c7 100644 --- a/golang/internal/actionparser/resources/invalid.yaml +++ b/golang/internal/actionparser/resources/invalid.yaml @@ -1,3 +1,6 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + name: Java CI on: [push, pull_request] diff --git a/golang/internal/actionparser/resources/valid.yaml b/golang/internal/actionparser/resources/valid.yaml index dfe411fd7..997096c4f 100644 --- a/golang/internal/actionparser/resources/valid.yaml +++ b/golang/internal/actionparser/resources/valid.yaml @@ -1,3 +1,6 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + name: Java CI on: [push, pull_request] @@ -12,22 +15,22 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v2 - with: - java-version: 8 - distribution: 'temurin' - cache: 'maven' + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + java-version: 8 + distribution: temurin + cache: maven - - name: Build with Maven - run: mvn verify -e -B -V -DdistributionFileName=apache-maven + - name: Build with Maven + run: mvn verify -e -B -V -DdistributionFileName=apache-maven - - name: Upload built Maven - uses: actions/upload-artifact@v2 - if: ${{ matrix.os == 'ubuntu-latest' }} - with: - name: built-maven - path: apache-maven/target/ + - name: Upload built Maven + uses: actions/upload-artifact@v2 + if: ${{ matrix.os == 'ubuntu-latest' }} + with: + name: built-maven + path: apache-maven/target/ integration-test: needs: build @@ -40,65 +43,65 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Collect environment context variables - shell: bash - env: - PR_HEAD_LABEL: ${{ github.event.pull_request.head.label }} - run: | - set +e - repo=maven-integration-testing - target_branch=master - target_user=apache - if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then - user=${PR_HEAD_LABEL%:*} - branch=${PR_HEAD_LABEL#*:} - else - user=${GITHUB_REPOSITORY%/*} - branch=${GITHUB_REF#refs/heads/} - fi - if [ $branch != "master" ]; then - git ls-remote https://github.com/$user/$repo.git | grep "refs/heads/${branch}$" > /dev/null - if [ $? -eq 0 ]; then - echo "Found a branch \"$branch\" in fork \"$user/$repo\", configuring this for the integration tests to be run against." - target_branch=$branch - target_user=$user - else - echo "Could not find fork \"$user/$repo\" or a branch \"$branch\" in this fork. Falling back to \"$target_branch\" in \"$target_user/$repo\"." - fi + - name: Collect environment context variables + shell: bash + env: + PR_HEAD_LABEL: ${{ github.event.pull_request.head.label }} + run: | + set +e + repo=maven-integration-testing + target_branch=master + target_user=apache + if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + user=${PR_HEAD_LABEL%:*} + branch=${PR_HEAD_LABEL#*:} + else + user=${GITHUB_REPOSITORY%/*} + branch=${GITHUB_REF#refs/heads/} + fi + if [ $branch != "master" ]; then + git ls-remote https://github.com/$user/$repo.git | grep "refs/heads/${branch}$" > /dev/null + if [ $? -eq 0 ]; then + echo "Found a branch \"$branch\" in fork \"$user/$repo\", configuring this for the integration tests to be run against." + target_branch=$branch + target_user=$user else - echo "Integration tests will run against $target_user/$repo for master builds." + echo "Could not find fork \"$user/$repo\" or a branch \"$branch\" in this fork. Falling back to \"$target_branch\" in \"$target_user/$repo\"." fi - echo "REPO_BRANCH=$target_branch" >> $GITHUB_ENV - echo "REPO_USER=$target_user" >> $GITHUB_ENV + else + echo "Integration tests will run against $target_user/$repo for master builds." + fi + echo "REPO_BRANCH=$target_branch" >> $GITHUB_ENV + echo "REPO_USER=$target_user" >> $GITHUB_ENV - - name: Checkout maven-integration-testing - uses: actions/checkout@v2 - with: - repository: ${{ env.REPO_USER }}/maven-integration-testing - path: maven-integration-testing/ - ref: ${{ env.REPO_BRANCH }} + - name: Checkout maven-integration-testing + uses: actions/checkout@v2 + with: + repository: ${{ env.REPO_USER }}/maven-integration-testing + path: maven-integration-testing/ + ref: ${{ env.REPO_BRANCH }} - - name: Set up cache for ~/.m2/repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: it-m2-repo-${{ matrix.os }}-${{ hashFiles('maven-integration-testing/**/pom.xml') }} - restore-keys: | - it-m2-repo-${{ matrix.os }}- + - name: Set up cache for ~/.m2/repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: it-m2-repo-${{ matrix.os }}-${{ hashFiles('maven-integration-testing/**/pom.xml') }} + restore-keys: | + it-m2-repo-${{ matrix.os }}- - - name: Download built Maven - uses: actions/download-artifact@v2 - with: - name: built-maven - path: built-maven/ + - name: Download built Maven + uses: actions/download-artifact@v2 + with: + name: built-maven + path: built-maven/ - - name: Set up JDK - uses: actions/setup-java@v2 - with: - java-version: ${{ matrix.java }} - distribution: 'temurin' - cache: 'maven' + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.java }} + distribution: temurin + cache: maven - - name: Running integration tests - shell: bash - run: mvn install -e -B -V -Prun-its,embedded -Dmaven.repo.local="$HOME/.m2/repository" -DmavenDistro="$GITHUB_WORKSPACE/built-maven/apache-maven-bin.zip" -f maven-integration-testing/pom.xml + - name: Running integration tests + shell: bash + run: mvn install -e -B -V -Prun-its,embedded -Dmaven.repo.local="$HOME/.m2/repository" -DmavenDistro="$GITHUB_WORKSPACE/built-maven/apache-maven-bin.zip" -f maven-integration-testing/pom.xml diff --git a/pyproject.toml b/pyproject.toml index dfa0bad22..6d0db25f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,233 @@ # Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. +# https://flit.pypa.io/en/latest/pyproject_toml.html +[build-system] +requires = ["flit_core >=3.2.0,<4.0.0"] +build-backend = "flit_core.buildapi" + + [project] name = "macaron" -requires-python = ">=3.10" -version = "0.0.0" -description = "Macaron" +requires-python = ">=3.11" authors = [ {"name" = "Trong Nhan Mai", "email" = "trong.nhan.mai@oracle.com"}, {"name" = "Behnaz Hassanshahi", "email" = "behnaz.hassanshahi@oracle.com"}, ] +maintainers = [ + {"name" = "Trong Nhan Mai", "email" = "trong.nhan.mai@oracle.com"}, + {"name" = "Behnaz Hassanshahi", "email" = "behnaz.hassanshahi@oracle.com"}, +] +dynamic = ["version", "description"] +license = {file = "LICENSE.txt"} +readme = "README.md" dependencies = [ - "requests ==2.28.0", - "pydriller >=2.0", - "yamale >=4.0.3", - "packaging ==21.3", - "jinja2 >=3.1.2" + "requests >=2.28.0,<3.0.0", + "pydriller >=2.0,<3.0.0", + "yamale >=4.0.3,<5.0.0", + "packaging >=21.3,<22.0.0", + "jinja2 >=3.1.2,<4.0.0" +] +keywords = [] +# https://pypi.org/classifiers/ +classifiers = [ + "Development Status :: 1 - Planning", + "Intended Audience :: Developers", + "License :: OSI Approved :: Universal Permissive License (UPL)", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Libraries :: Python Modules", ] [project.scripts] -macaron = 'macaron.__main__:main' \ No newline at end of file +macaron = 'macaron.__main__:main' + +[project.entry-points] + +[project.optional-dependencies] +# The 'actions' requirements match exactly the packages installed by the workflows. +# We keep them listed here to ensure the infrastructure BOM is consistent with what's +# installed. Make sure to keep the requirements in sync with the workflows! +actions = [ + "commitizen >=2.37.1,<3.0.0", + "twine >=4.0.1,<5.0.0", +] +dev = [ + "flit >=3.2.0,<4.0.0", + "mypy >=0.921,<0.992", + "types-pyyaml >=6.0.4,<7.0.0", + "types-requests >=2.25.6,<3.0.0", + # Exclude pip-audit v2.4.9 because it has a bug. + # See https://github.com/pypa/pip-audit/commit/22d7e4c7f5acd20852c57b52b46e861a716ab09f. + "pip-audit >=2.4.8,<3.0.0,!=2.4.9", + "pylint >=2.9.3,<2.15.8", + "cyclonedx-bom >=3.5.0,<4.0.0", +] +docs = [ + "sphinx >=5.3.0,<6.0.0", + "sphinx-autodoc-typehints >=1.19.4,<2.0.0", + "sphinx-rtd-theme >=1.0.0,<2.0.0", + "numpydoc >=1.5.0,<2.0.0", +] +hooks = [ + "pre-commit >=2.18.0,<=2.20.0", +] +# Note that the `custom_exit_code` and `env` plugins may currently be unmaintained. +test = [ + "hypothesis >=6.21.0,<6.58.2", + "pytest >=7.2.0,<8.0.0", + "pytest-custom_exit_code >=0.3.0,<1.0.0", + "pytest-cov >=4.0.0,<5.0.0", + "pytest-env >=0.8.1,<1.0.0", +] + +[project.urls] +Homepage = "https://my.project/" +Changelog = "https://my.project/CHANGELOG" +Documentation = "https://my.project/docs/" +Issues = "https://my.project/issues" + + +# https://bandit.readthedocs.io/en/latest/config.html +# Skip test B101 because of issue https://github.com/PyCQA/bandit/issues/457 +[tool.bandit] +tests = [] +skips = ["B101"] + + +# https://github.com/psf/black#configuration +[tool.black] +line-length = 120 + + +# https://github.com/commitizen-tools/commitizen +# https://commitizen-tools.github.io/commitizen/bump/ +[tool.commitizen] +bump_message = """bump: release $current_version → $new_version + +Automatically generated by Commitizen. +""" +tag_format = "v$major.$minor.$patch$prerelease" +update_changelog_on_bump = true +version_files = [ + "src/macaron/__init__.py:__version__", +] +major_version_zero = false +version = "2.6.0" + + +# https://github.com/pytest-dev/pytest-cov +# https://github.com/nedbat/coveragepy +[tool.coverage.report] +fail_under = 60 +show_missing = true + +[tool.coverage.run] +omit = [ + "src/macaron/__main__.py", +] + + +# https://flit.pypa.io/en/latest/pyproject_toml.html#sdist-section +# See also: https://github.com/pypa/flit/issues/565 +[tool.flit.sdist] +include = [] +exclude = [ + ".github/", + "docs/", + "tests/", + ".flake8", + ".gitignore", + ".pre-commit-config.yaml", + "CHANGELOG.md", + "Makefile", + "SECURITY.md", +] + + +# https://pycqa.github.io/isort/ +[tool.isort] +profile = "black" +multi_line_output = 3 +line_length = 120 +skip_gitignore = true + + +# https://mypy.readthedocs.io/en/stable/config_file.html#using-a-pyproject-toml +[tool.mypy] +mypy_path = "src/macaron/:tests/" +# exclude= +show_error_codes = true +show_column_numbers = true +check_untyped_defs = true +incremental = false +strict_equality = true +warn_return_any = true +warn_redundant_casts = true +warn_unreachable = true +warn_unused_configs = true +warn_unused_ignores = true +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +disallow_untyped_decorators = true +disable_error_code = [] + +[[tool.mypy.overrides]] +module = [ + "pytest.*", + "pydriller.*", + "gitdb.*", + "yamale.*", +] +ignore_missing_imports = true + + +# https://pylint.pycqa.org/en/latest/user_guide/configuration/index.html +[tool.pylint.MASTER] +fail-under = 10.0 +disable = [ + "fixme", + "too-few-public-methods", + "too-many-ancestors", + "too-many-arguments", + "too-many-boolean-expressions", + "too-many-branches", + "too-many-instance-attributes", + "too-many-lines", + "too-many-locals", + "too-many-nested-blocks", + "too-many-public-methods", + "too-many-return-statements", + "too-many-statements", + "duplicate-code", +] + +[tool.pylint.MISCELLANEOUS] +notes = [ + "FIXME", + "TODO", + "BUGBUG", +] + +[tool.pylint.FORMAT] +max-line-length = 120 + + +# https://docs.pytest.org/en/latest/reference/customize.html#configuration-file-formats +# https://docs.pytest.org/en/latest/reference/reference.html#configuration-options +# https://docs.pytest.org/en/latest/reference/reference.html#command-line-flags +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "--verbose --doctest-modules -ra --cov macaron" # Consider adding --pdb +doctest_optionflags = "IGNORE_EXCEPTION_DETAIL" +testpaths = [ + "tests", +] +env = [ + "PYTHONWARNINGS=always::DeprecationWarning", +] diff --git a/scripts/dev_scripts/build.sh b/scripts/dev_scripts/build.sh index 9916e504f..3175b9e27 100755 --- a/scripts/dev_scripts/build.sh +++ b/scripts/dev_scripts/build.sh @@ -1,4 +1,5 @@ #!/bin/sh + # Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/scripts/dev_scripts/copyright-checker.sh b/scripts/dev_scripts/copyright-checker.sh new file mode 100755 index 000000000..88233a58f --- /dev/null +++ b/scripts/dev_scripts/copyright-checker.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# +# Checks if copyright header is valid +# + +files=$(git diff --cached --name-only) +currentyear=$(date +"%Y") +missing_copyright_files=() + + +for f in $files; do + if [ ! -f "$f" ]; then + continue + fi + startyear=$(git log --format=%ad --date=format:%Y "$f" | tail -1) + if [[ -z "${startyear// }" ]]; then + startyear=$currentyear + fi + if ! grep -i -e "Copyright (c) $startyear - $currentyear, Oracle and/or its affiliates. All rights reserved." "$f" 1>/dev/null;then + if [[ $f =~ .*\.(js$|py$|java$|tf$|go$|sh$|dl$|yaml$) ]] || [[ "${f##*/}" = "Dockerfile" ]];then + missing_copyright_files+=("$f") + fi + fi +done + +if [ ${#missing_copyright_files[@]} -ne 0 ]; then + for f in "${missing_copyright_files[@]}"; do + startyear=$(git log --format=%ad --date=format:%Y "$f" | tail -1) + if [[ -z "${startyear// }" ]]; then + startyear=$currentyear + fi + if [[ $f =~ .*\.(js$|java$|go$|dl$) ]]; then + expected="\/\* Copyright \(c\) $startyear - $currentyear, Oracle and\/or its affiliates\. All rights reserved\. \*\/" + expected="$expected\n\/\* Licensed under the Universal Permissive License v 1.0 as shown at https:\/\/oss\.oracle\.com\/licenses\/upl\/\. \*\/" + elif [[ $f =~ .*\.(py$|tf$|sh$|yaml$) ]] || [[ "${f##*/}" = "Dockerfile" ]]; then + expected="# Copyright \(c\) $startyear - $currentyear, Oracle and\/or its affiliates\. All rights reserved\." + expected="$expected\n# Licensed under the Universal Permissive License v 1.0 as shown at https:\/\/oss\.oracle\.com\/licenses\/upl\/\." + + fi + + if ! grep -i -e "Copyright (c) .* Oracle and/or its affiliates. All rights reserved" "$f" 1>/dev/null;then + echo "Copyright header missing for $f" + sed -i "1s/^/$expected\n\n/" "$f" + else + echo "Copyright header needs update for $f" + sed -i "1s/^.*/$expected/" "$f" + fi + done + echo "Copyright headers have been automatically added/updated. Please review and stage the changes before running git commit again." + exit 1 +fi diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index f601825f6..5d2145e81 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -8,7 +8,7 @@ WORKSPACE=$1 HOMEDIR=$2 COMPARE_DEPS=$WORKSPACE/tests/dependency_analyzer/compare_dependencies.py COMPARE_JSON_OUT=$WORKSPACE/tests/e2e/compare_e2e_result.py -RUN_MACARON="python -m macaron -o $WORKSPACE/output -t $GH_TOKEN" +RUN_MACARON="python -m macaron -o $WORKSPACE/output -t $GITHUB_TOKEN" RESULT_CODE=0 if [[ ! -d "$HOMEDIR/.m2/settings.xml" ]]; @@ -20,16 +20,11 @@ then cp $WORKSPACE/resources/settings.xml $HOMEDIR/.m2/ fi -Running Macaron without config files +# Running Macaron without config files echo -e "\n==================================================================================" echo "Run integration tests without configurations" echo -e "==================================================================================\n" -echo -e "\n----------------------------------------------------------------------------------" -echo "pmd/pmd: Analyze using only the repo path when automatic dependency resolution is skipped." -echo -e "----------------------------------------------------------------------------------\n" -$RUN_MACARON analyze -rp https://github.com/pmd/pmd --skip-deps || RESULT_CODE=1 - echo -e "\n----------------------------------------------------------------------------------" echo "micronaut-projects/micronaut-core: Analyzing the repo path and the branch name when automatic dependency resolution is skipped." echo -e "----------------------------------------------------------------------------------\n" @@ -63,11 +58,11 @@ $RUN_MACARON analyze -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 8 python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || RESULT_CODE=1 echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Analyzing the repo path, the branch name and the commit digest with dependency resolution using osint maven plugin (default)." +echo "apache/maven: Analyzing the repo path, the branch name and the commit digest with dependency resolution using cyclonedx maven plugin (default)." echo -e "----------------------------------------------------------------------------------\n" JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/osint_maven_apache_maven.json +DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json $RUN_MACARON analyze -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 || RESULT_CODE=1 @@ -84,15 +79,15 @@ DEP_RESULT=$WORKSPACE/output/reports/github_com/micronaut-projects/micronaut-cor echo -e "\n----------------------------------------------------------------------------------" echo "micronaut-projects/micronaut-core: Check the resolved dependency output when automatic dependency resolution is skipped." echo -e "----------------------------------------------------------------------------------\n" -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/osint_maven_micronaut-projects_micronaut-core.json +DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_micronaut-projects_micronaut-core.json $RUN_MACARON analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/micronaut_core_config.yaml --skip-deps || RESULT_CODE=1 python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || RESULT_CODE=1 echo -e "\n----------------------------------------------------------------------------------" -echo "micronaut-projects/micronaut-core: Check the resolved dependency output with config for osint maven plugin (default)." +echo "micronaut-projects/micronaut-core: Check the resolved dependency output with config for cyclonedx maven plugin (default)." echo -e "----------------------------------------------------------------------------------\n" -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/osint_maven_micronaut-projects_micronaut-core.json +DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_micronaut-projects_micronaut-core.json $RUN_MACARON analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/micronaut_core_config.yaml || RESULT_CODE=1 python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || RESULT_CODE=1 @@ -149,28 +144,14 @@ do python $COMPARE_JSON_OUT $JSON_RESULT_DIR/$i $JSON_EXPECT_DIR/$i || RESULT_CODE=1 done - -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Check the resolved dependency output with config for osint maven plugin (default)." -echo -e "----------------------------------------------------------------------------------\n" -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/osint_maven_apache_maven.json -$RUN_MACARON analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || RESULT_CODE=1 - -python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || RESULT_CODE=1 - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Check the resolved dependency output with config for cyclonedx maven plugin." echo -e "----------------------------------------------------------------------------------\n" -# Add the user defaults.ini that sets cyclonedx-maven to the root path. -cp $WORKSPACE/tests/config/resources/defaults.ini $WORKSPACE DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json $RUN_MACARON analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || RESULT_CODE=1 python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || RESULT_CODE=1 -# Remove the user defaults.ini to use osint-maven by default. -rm $WORKSPACE/defaults.ini - echo -e "\n----------------------------------------------------------------------------------" echo "apache/mavenCheck: Check the e2e status code of running with invalid branch or digest defined in the yaml configuration." echo -e "----------------------------------------------------------------------------------\n" @@ -215,9 +196,9 @@ $RUN_MACARON analyze -c $WORKSPACE/tests/e2e/configurations/jackson_databind_con python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || RESULT_CODE=1 # echo -e "\n----------------------------------------------------------------------------------" -# echo "FasterXML/jackson-databind: Check the resolved dependency output with config for osint maven plugin (default)." +# echo "FasterXML/jackson-databind: Check the resolved dependency output with config for cyclonedx maven plugin (default)." # echo -e "----------------------------------------------------------------------------------\n" -# DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/osint_maven_FasterXML_jackson-databind.json +# DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_FasterXML_jackson-databind.json # DEP_RESULT=$WORKSPACE/output/reports/github_com/FasterXML/jackson-databind/dependencies.json # $RUN_MACARON analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/jackson_databind_config.yaml || RESULT_CODE=1 @@ -229,11 +210,11 @@ echo "Run integration tests with local paths for apache/maven..." echo -e "==================================================================================\n" echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Analyzing with the branch name, the commit digest and dependency resolution using osint maven plugin (default)." +echo "apache/maven: Analyzing with the branch name, the commit digest and dependency resolution using cyclonedx maven plugin (default)." echo -e "----------------------------------------------------------------------------------\n" JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/osint_maven_apache_maven.json +DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json $RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com analyze -rp apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 || RESULT_CODE=1 @@ -357,5 +338,6 @@ fi if [ $RESULT_CODE -ne 0 ]; then + echo -e "Expected zero status code but got $RESULT_CODE." exit 1 fi diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index e1716cd24..623ead58e 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -24,10 +24,10 @@ COMPARE_JSON_OUT=$WORKSPACE/tests/e2e/compare_e2e_result.py RESULT_CODE=0 echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Check the resolved dependency output with config for osint maven plugin (default)." +echo "apache/maven: Check the resolved dependency output with config for cyclonedx maven plugin (default)." echo -e "----------------------------------------------------------------------------------\n" DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/osint_maven_apache_maven.json +DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json $RUN_MACARON -C $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || RESULT_CODE=1 $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || RESULT_CODE=1 diff --git a/src/macaron/__init__.py b/src/macaron/__init__.py index a13978d0d..b19f2502a 100644 --- a/src/macaron/__init__.py +++ b/src/macaron/__init__.py @@ -2,3 +2,8 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module initializes the necessary components for the macaron package.""" + +# The version of this package. There's no comprehensive, official list of other +# magic constants, so we stick with this one only for now. See also this conversation: +# https://stackoverflow.com/questions/38344848/is-there-a-comprehensive-table-of-pythons-magic-constants +__version__ = "0.0.0" diff --git a/src/macaron/code_analyzer/call_graph.py b/src/macaron/code_analyzer/call_graph.py index 4be8f613f..0a9a49e10 100644 --- a/src/macaron/code_analyzer/call_graph.py +++ b/src/macaron/code_analyzer/call_graph.py @@ -3,7 +3,8 @@ """This module contains classes to generate build call graphs for the target repository.""" -from typing import Generic, Iterable, TypeVar +from collections.abc import Iterable +from typing import Generic, TypeVar Node = TypeVar("Node", bound="BaseNode") """This binds type ``Node`` to ``BaseNode`` and any of its subclasses. @@ -17,6 +18,7 @@ class BaseNode(Generic[Node]): """This is the generic class for call graph nodes.""" def __init__(self) -> None: + """Initialize instance.""" self.callee: list[Node] = [] def add_callee(self, node: Node) -> None: @@ -41,17 +43,18 @@ def has_callee(self) -> bool: class CallGraph(Generic[Node]): - """This is the generic class for creating a call graph. - - Parameters - ---------- - root : Node - The root call graph node. - repo_path : str - The path to the repo. - """ + """This is the generic class for creating a call graph.""" def __init__(self, root: Node, repo_path: str) -> None: + """Initialize instance. + + Parameters + ---------- + root : Node + The root call graph node. + repo_path : str + The path to the repo. + """ self.root = root self.repo_path = repo_path diff --git a/src/macaron/config/defaults.ini b/src/macaron/config/defaults.ini index e18d5b3e5..7e7519f45 100644 --- a/src/macaron/config/defaults.ini +++ b/src/macaron/config/defaults.ini @@ -6,24 +6,24 @@ runner_num = 1 # that runner will be put back into the queue to keep running if it hasn't finished. timeout = 5 +[requests] +# The default timeout in seconds for 'requests' API calls. +timeout = 10 # This is the database to store Macaron's results. [database] db_name = macaron.db - # This is the parser for GitHub Actions workflows. [actionparser] # This is the timeout (in seconds) for the actionparser. timeout = 30 - # This is the parser for bash scripts. [bashparser] # This is the timeout (in seconds) for the bashparser. timeout = 30 - # This is the dependency resolver tool to generate SBOM. [dependency.resolver] # Should be in : format. @@ -33,7 +33,6 @@ dep_tool_maven = cyclonedx-maven:2.6.2 # This is the timeout (in seconds) to run the dependency resolver. timeout = 1200 - [git] # The list of allowed git hosts. # Host names are separated by spaces and they can be defined in multiple lines. @@ -43,7 +42,6 @@ allowed_hosts = ol-bitbucket.us.oracle.com gitlab.com - # This is the spec for trusted Maven build tools. [builder.maven] entry_conf = settings.xml @@ -144,7 +142,6 @@ jenkins = org.sonatype.plugins:nexus-staging-maven-plugin:deploy-staged-repository nxrm3:staging-deploy - # This is the spec for trusted Gradle build tools. [builder.gradle] entry_conf = @@ -224,28 +221,24 @@ max_workflow_persist = 90 entry_conf = Jenkinsfile - # This is the spec for Travis CI. [ci.travis_ci] entry_conf = .travis.yml .travis.yaml - # This is the spec for Circle CI. [ci.circle_ci] entry_conf = .circleci/config.yml .circleci/config.yaml - # This is the spec for GitLab CI. [ci.gitlab_ci] entry_conf = .gitlab-ci.yml .gitlab-ci.yaml - # Configuration options for SLSA verifier. [slsa.verifier] provenance_extensions = diff --git a/src/macaron/config/defaults.py b/src/macaron/config/defaults.py index 09bae431b..e098ecd3f 100644 --- a/src/macaron/config/defaults.py +++ b/src/macaron/config/defaults.py @@ -8,6 +8,7 @@ import os import pathlib import shutil +from typing import Optional logger: logging.Logger = logging.getLogger(__name__) @@ -16,7 +17,12 @@ class ConfigParser(configparser.ConfigParser): """This class extends ConfigParser with useful methods.""" def get_list( - self, section: str, item: str, delimiter: str = None, fallback: list = None, duplicated_ok: bool = False + self, + section: str, + item: str, + delimiter: Optional[str] = None, + fallback: Optional[list] = None, + duplicated_ok: bool = False, ) -> list: """Parse and return a list of strings from an item in ``defaults.ini``. @@ -34,9 +40,9 @@ def get_list( The section in ``defaults.ini``. item : str The item to parse the list. - delimiter : str + delimiter : Optional[str] The delimiter used to split the strings. - fallback : list + fallback : Optional[list] The fallback value in case of errors. duplicated_ok : bool If True allow duplicate values. @@ -70,13 +76,11 @@ def get_list( distinct_values = set() distinct_values.update(content) - return list(distinct_values) - - return fallback or [] except configparser.NoOptionError as error: logger.error(error) - return fallback or [] + + return fallback or [] defaults = ConfigParser() diff --git a/src/macaron/config/target_config.py b/src/macaron/config/target_config.py index dd941e4ec..e169802b4 100644 --- a/src/macaron/config/target_config.py +++ b/src/macaron/config/target_config.py @@ -5,7 +5,7 @@ import logging import os -from typing import Any +from typing import Any, Optional import yamale from yamale.schema import Schema @@ -21,12 +21,12 @@ class Configuration: """This class contains the configuration for an analyzed repo in Macaron.""" - def __init__(self, data: dict = None) -> None: + def __init__(self, data: Optional[dict] = None) -> None: """Construct the Configuration object. Parameters ---------- - data : dict + data : Optional[dict] The dictionary contains the data to analyze a repository. """ self.options = {"id": "", "path": "", "branch": "", "digest": "", "note": "", "available": ""} diff --git a/src/macaron/config/target_config_schema.yaml b/src/macaron/config/target_config_schema.yaml index 96c0ada7d..dd9463cfc 100644 --- a/src/macaron/config/target_config_schema.yaml +++ b/src/macaron/config/target_config_schema.yaml @@ -1,3 +1,4 @@ +--- # Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/src/macaron/database/database_manager.py b/src/macaron/database/database_manager.py index a8b1429ed..b609f04eb 100644 --- a/src/macaron/database/database_manager.py +++ b/src/macaron/database/database_manager.py @@ -10,15 +10,16 @@ class DatabaseManager: - """This class handles and manages the connection to sqlite database during the search session. - - Parameters - ---------- - db_path : str - The path to the target database. - """ + """This class handles and manages the connection to sqlite database during the search session.""" def __init__(self, db_path: str): + """Initialize instance. + + Parameters + ---------- + db_path : str + The path to the target database. + """ self.db_path = db_path self.is_init = False self.db_con = None @@ -77,7 +78,7 @@ def execute_select_query(self, query: str) -> list: """ logger.debug("Executing DB query: %s", query) try: - result = self.db_cursor.execute(query).fetchall() # type: ignore + result: list = self.db_cursor.execute(query).fetchall() # type: ignore return result except sqlite3.OperationalError as error: logger.error( diff --git a/src/macaron/dependency_analyzer/__init__.py b/src/macaron/dependency_analyzer/__init__.py index c989ccf5c..81adbe19e 100644 --- a/src/macaron/dependency_analyzer/__init__.py +++ b/src/macaron/dependency_analyzer/__init__.py @@ -3,5 +3,5 @@ """This package contains the dependency resolvers for Java projects.""" -from .cyclonedx_mvn import CycloneDxMaven -from .dependency_resolver import DependencyAnalyzer, DependencyInfo, DependencyTools +from .cyclonedx_mvn import CycloneDxMaven # noqa: F401 +from .dependency_resolver import DependencyAnalyzer, DependencyInfo, DependencyTools # noqa: F401 diff --git a/src/macaron/dependency_analyzer/dependency_resolver.py b/src/macaron/dependency_analyzer/dependency_resolver.py index 0a7153ba1..aa3116d7e 100644 --- a/src/macaron/dependency_analyzer/dependency_resolver.py +++ b/src/macaron/dependency_analyzer/dependency_resolver.py @@ -6,7 +6,7 @@ import logging from abc import abstractmethod from enum import Enum -from typing import Optional, TypedDict # pylint: disable=unused-import +from typing import TypedDict from packaging import version @@ -190,7 +190,7 @@ def merge_configs( dep.set_value("available", SCMStatus.AVAILABLE) merged_deps.append(dep) - if resolved_deps is None: + if not resolved_deps: return merged_deps for key, value in resolved_deps.items(): diff --git a/src/macaron/output_reporter/jinja2_extensions.py b/src/macaron/output_reporter/jinja2_extensions.py index 7245695e3..620a2ea20 100644 --- a/src/macaron/output_reporter/jinja2_extensions.py +++ b/src/macaron/output_reporter/jinja2_extensions.py @@ -206,7 +206,7 @@ def j2_filter_get_check_result_color(result_type: str) -> str: filter_extensions: dict[str, str] = { - filter.replace("j2_filter_", ""): filter for filter in dir() if filter.startswith("j2_filter_") + filter_str.replace("j2_filter_", ""): filter_str for filter_str in dir() if filter_str.startswith("j2_filter_") } """The mappings between the name of a filter and its function's name as defined in this module.""" diff --git a/src/macaron/output_reporter/reporter.py b/src/macaron/output_reporter/reporter.py index 04d98e2da..d79fc5f32 100644 --- a/src/macaron/output_reporter/reporter.py +++ b/src/macaron/output_reporter/reporter.py @@ -7,6 +7,7 @@ import logging import os from copy import deepcopy +from typing import Optional from jinja2 import ( Environment, @@ -26,17 +27,18 @@ class FileReporter: - """The reporter that handles writing data to disk files. - - Parameters - ---------- - mode : str, optional - The mode to open the target files, by default "w". - encoding : str, optional - The encoding used to handle disk files, by default "utf-8". - """ + """The reporter that handles writing data to disk files.""" def __init__(self, mode: str = "w", encoding: str = "utf-8"): + """Initialize instance. + + Parameters + ---------- + mode : str, optional + The mode to open the target files, by default "w". + encoding : str, optional + The encoding used to handle disk files, by default "utf-8". + """ self.mode = mode self.encoding = encoding @@ -80,15 +82,20 @@ def generate(self, target_dir: str, report: Report) -> None: class JSONReporter(FileReporter): - """This class handles writing reports to JSON files. - - Parameters - ---------- - indent : int, optional - The indent for the JSON output, by default 4. - """ + """This class handles writing reports to JSON files.""" def __init__(self, mode: str = "w", encoding: str = "utf-8", indent: int = 4): + """Initialize instance. + + Parameters + ---------- + mode: str, optional + The file operation mode. + encoding: str, optional + The encoding. + indent : int, optional + The indent for the JSON output, by default 4. + """ super().__init__(mode, encoding) self.indent = indent @@ -122,20 +129,29 @@ def generate(self, target_dir: str, report: Report) -> None: class HTMLReporter(FileReporter): - """This class handles writing reports to HTML files. - - Parameters - ---------- - env : Environment - The pre-initiated ``jinja2.Environment`` instance for the HTMLReporter. If this is not - provided, a default jinja2.Environment will be initialized. - target_template : str - The target template. It will be looked up from the jinja2.Environment instance. - """ + """This class handles writing reports to HTML files.""" def __init__( - self, mode: str = "w", encoding: str = "utf-8", env: Environment = None, target_template: str = "macaron.html" + self, + mode: str = "w", + encoding: str = "utf-8", + env: Optional[Environment] = None, + target_template: str = "macaron.html", ) -> None: + """Initialize instance. + + Parameters + ---------- + mode: str, optional + The file operation mode. + encoding: str, optional + The encoding. + env : Optional[Environment] + The pre-initiated ``jinja2.Environment`` instance for the HTMLReporter. If this is not + provided, a default jinja2.Environment will be initialized. + target_template : str + The target template. It will be looked up from the jinja2.Environment instance. + """ super().__init__(mode, encoding) if env: self.env = env diff --git a/src/macaron/output_reporter/results.py b/src/macaron/output_reporter/results.py index ba6098662..0f261740d 100644 --- a/src/macaron/output_reporter/results.py +++ b/src/macaron/output_reporter/results.py @@ -3,10 +3,11 @@ """This module contains classes that represent the result of the Macaron analysis.""" +from collections.abc import Iterable from dataclasses import dataclass, field from datetime import datetime from enum import Enum -from typing import Generic, Iterable, TypedDict, TypeVar +from typing import Generic, TypedDict, TypeVar from macaron.config.target_config import Configuration from macaron.slsa_analyzer.analyze_context import AnalyzeContext @@ -168,15 +169,16 @@ def get_dep_summary(self) -> DepSummary: class Report: - """This class contains the report content of an analysis. - - Parameters - ---------- - root_record : Record - The record of the main target repository. - """ + """This class contains the report content of an analysis.""" def __init__(self, root_record: Record) -> None: + """Initialize instance. + + Parameters + ---------- + root_record : Record + The record of the main target repository. + """ # The record of the target repo in the analysis. self.root_record: Record = root_record self.record_mapping: dict[str, Record] = {} diff --git a/src/macaron/parsers/actionparser.py b/src/macaron/parsers/actionparser.py index 4b728ee0a..068ac5d59 100644 --- a/src/macaron/parsers/actionparser.py +++ b/src/macaron/parsers/actionparser.py @@ -20,7 +20,7 @@ logger: logging.Logger = logging.getLogger(__name__) -def parse(workflow_path: str, macaron_path: str = None) -> dict: +def parse(workflow_path: str, macaron_path: str = "") -> dict: """Parse the GitHub Actions workflow YAML file. Parameters @@ -35,7 +35,7 @@ def parse(workflow_path: str, macaron_path: str = None) -> dict: dict The parsed workflow as a JSON (dict) object. """ - if macaron_path is None: + if not macaron_path: macaron_path = global_config.macaron_path cmd = [ os.path.join(macaron_path, "bin", "actionparser"), @@ -61,7 +61,7 @@ def parse(workflow_path: str, macaron_path: str = None) -> dict: try: if result.returncode == 0: - parsed_obj = json.loads(result.stdout.decode("utf-8")) + parsed_obj: dict = json.loads(result.stdout.decode("utf-8")) return parsed_obj logger.error("GitHub Actions parser failed: %s", result.stderr) return {} diff --git a/src/macaron/parsers/bashparser.py b/src/macaron/parsers/bashparser.py index 720ec6d14..f7b03d9f5 100644 --- a/src/macaron/parsers/bashparser.py +++ b/src/macaron/parsers/bashparser.py @@ -13,7 +13,8 @@ import logging import os import subprocess # nosec B404 -from typing import Iterable, TypedDict +from collections.abc import Iterable +from typing import TypedDict from macaron.config.defaults import defaults from macaron.config.global_config import global_config @@ -34,7 +35,7 @@ class BashCommands(TypedDict): """Parsed bash commands.""" -def parse_file(file_path: str, macaron_path: str = None) -> dict: +def parse_file(file_path: str, macaron_path: str = "") -> dict: """Parse a bash script file. Parameters @@ -49,7 +50,7 @@ def parse_file(file_path: str, macaron_path: str = None) -> dict: dict The parsed bash script in JSON (dict) format. """ - if macaron_path is None: + if not macaron_path: macaron_path = global_config.macaron_path try: with open(file_path, encoding="utf8") as file: @@ -60,7 +61,7 @@ def parse_file(file_path: str, macaron_path: str = None) -> dict: return {} -def parse(bash_content: str, macaron_path: str = None) -> dict: +def parse(bash_content: str, macaron_path: str = "") -> dict: """Parse a bash script's content. Parameters @@ -75,7 +76,7 @@ def parse(bash_content: str, macaron_path: str = None) -> dict: dict The parsed bash script in JSON (dict) format. """ - if macaron_path is None: + if not macaron_path: macaron_path = global_config.macaron_path cmd = [ os.path.join(macaron_path, "bin", "bashparser"), @@ -101,7 +102,7 @@ def parse(bash_content: str, macaron_path: str = None) -> dict: try: if result.returncode == 0: - return json.loads(result.stdout.decode("utf-8")) + return dict(json.loads(result.stdout.decode("utf-8"))) logger.error("Bash script parser failed: %s", result.stderr) return {} @@ -114,7 +115,7 @@ def extract_bash_from_ci( bash_content: str, ci_file: str, ci_type: str, - macaron_path: str = None, + macaron_path: str = "", recursive: bool = False, repo_path: str = "", working_dir: str = "", @@ -145,7 +146,7 @@ def extract_bash_from_ci( BashCommands The parsed bash script objects. """ - if macaron_path is None: + if not macaron_path: macaron_path = global_config.macaron_path parsed_parent = parse(bash_content) diff --git a/src/macaron/parsers/yaml/loader.py b/src/macaron/parsers/yaml/loader.py index 1a2140df5..30712caa0 100644 --- a/src/macaron/parsers/yaml/loader.py +++ b/src/macaron/parsers/yaml/loader.py @@ -38,12 +38,12 @@ def _load_yaml_content(path: os.PathLike | str) -> list: """ try: logger.debug("Loading yaml from file %s", path) - return yamale.make_data(path) + return list(yamale.make_data(path)) except YAMLError as error: abs_path = os.path.abspath(path) if hasattr(error, "problem_mark"): - mark = error.problem_mark # type: ignore + mark = error.problem_mark line_number = mark.line + 1 column_number = mark.column + 1 err_pos = f"{line_number}:{column_number}" diff --git a/src/macaron/policy_engine/policy.py b/src/macaron/policy_engine/policy.py index 4d62ebc63..636ff1ebc 100644 --- a/src/macaron/policy_engine/policy.py +++ b/src/macaron/policy_engine/policy.py @@ -7,7 +7,7 @@ import os from dataclasses import dataclass, field from functools import reduce -from typing import Any, Callable, Generic, TypeVar, Union +from typing import Any, Callable, Generic, Optional, TypeVar, Union import yamale from yamale.schema import Schema @@ -22,7 +22,7 @@ SubscriptPathType = list[Union[str, int]] Primitive = Union[str, bool, int, float] -PolicyDef = Union[Primitive, dict, list] +PolicyDef = Union[Primitive, dict, list, None] PolicyFn = Callable[[Any], bool] GPolicy = TypeVar("GPolicy", bound="Policy") @@ -81,14 +81,14 @@ def _get_path_as_str(path: SubscriptPathType) -> str: return result.lstrip(".") -def _gen_policy_func(policy: PolicyDef, path: SubscriptPathType = None) -> PolicyFn: +def _gen_policy_func(policy: PolicyDef, path: Optional[SubscriptPathType] = None) -> PolicyFn: """Get the policy verify function from the policy data. Parameters ---------- policy : PolicyType The policy data. - path : SubscriptPathType + path : Optional[SubscriptPathType] Describe the path taken to get to the current ``policy``. Returns @@ -101,7 +101,7 @@ def _gen_policy_func(policy: PolicyDef, path: SubscriptPathType = None) -> Polic InvalidPolicyError If the provided policy is invalid. """ - res_path: SubscriptPathType = [] if not path else path + res_path: SubscriptPathType = path or [] match policy: case str() | bool() | int() | float(): logger.debug("%sPrimitive(policy=%s, path=%s)", "\t" * len(res_path), str(policy), res_path) diff --git a/src/macaron/py.typed b/src/macaron/py.typed new file mode 100644 index 000000000..65cce6fa0 --- /dev/null +++ b/src/macaron/py.typed @@ -0,0 +1,4 @@ +# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +# PEP-561 marker. https://mypy.readthedocs.io/en/latest/installed_packages.html diff --git a/src/macaron/slsa_analyzer/analyze_context.py b/src/macaron/slsa_analyzer/analyze_context.py index e791ba4a1..5cf1f4dec 100644 --- a/src/macaron/slsa_analyzer/analyze_context.py +++ b/src/macaron/slsa_analyzer/analyze_context.py @@ -41,29 +41,7 @@ class ChecksOutputs(TypedDict): class AnalyzeContext: - """This class contains data of the current analyzed repository. - - Parameters - ---------- - full_name : str - Repository name in ``/`` format. - repo_path : str - Target repository path. - git_obj : Git - The Git object for the target path. - branch_name : str - The target branch. - commit_sha : str - The commit sha of the target repo. - commit_date : str - The commit date of the target repo. - macaron_path : str - The Macaron's root path. - output_dir : str - The output dir. - remote_path : str - The remote path for the target repo. - """ + """This class contains data of the current analyzed repository.""" def __init__( self, @@ -77,6 +55,29 @@ def __init__( output_dir: str = "", remote_path: str = "", ): + """Initialize instance. + + Parameters + ---------- + full_name : str + Repository name in ``/`` format. + repo_path : str + Target repository path. + git_obj : Git + The Git object for the target path. + branch_name : str + The target branch. + commit_sha : str + The commit sha of the target repo. + commit_date : str + The commit date of the target repo. + macaron_path : str + The Macaron's root path. + output_dir : str + The output dir. + remote_path : str + The remote path for the target repo. + """ # / self.repo_full_name = full_name diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 3fb32def6..7d0c8dedc 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -8,6 +8,7 @@ import subprocess # nosec B404 import sys from datetime import datetime +from typing import Optional from git import InvalidGitRepositoryError from pydriller.git import Git @@ -26,7 +27,7 @@ from macaron.slsa_analyzer.build_tool.maven import Maven # To load all checks into the registry -from macaron.slsa_analyzer.checks import * # pylint: disable=wildcard-import,unused-wildcard-import +from macaron.slsa_analyzer.checks import * # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F401,F403 from macaron.slsa_analyzer.checks.check_result import CheckResult, SkippedInfo from macaron.slsa_analyzer.ci_service import CI_SERVICES from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService @@ -39,15 +40,7 @@ class Analyzer: - """This class is used to analyze SLSA levels of a Git repo. - - Parameters - ---------- - output_path : str - The path to the output directory. - build_log_path : str - The path to store the build logs. - """ + """This class is used to analyze SLSA levels of a Git repo.""" GIT_REPOS_DIR = "git_repos" """The directory in the output dir to store all cloned repositories.""" @@ -56,6 +49,15 @@ class Analyzer: """The name of the SQLite table which stores the analyze results.""" def __init__(self, output_path: str, build_log_path: str) -> None: + """Initialize instance. + + Parameters + ---------- + output_path : str + The path to the output directory. + build_log_path : str + The path to store the build logs. + """ if not os.path.isdir(output_path): logger.critical("%s is not a valid directory. Exiting ...", output_path) sys.exit(1) @@ -238,7 +240,7 @@ def resolve_dependencies(self, main_ctx: AnalyzeContext) -> dict[str, Dependency ).split(":") ) if tool_name == DependencyTools.CYCLONEDX_MAVEN: - dep_analyzer = CycloneDxMaven( # type: ignore + dep_analyzer = CycloneDxMaven( resources_path=global_config.resources_path, file_name="bom.json", debug_path=os.path.join(self.output_path, "cdx_debug.json"), @@ -301,7 +303,7 @@ def resolve_dependencies(self, main_ctx: AnalyzeContext) -> dict[str, Dependency return deps_resolved - def run_single(self, config: Configuration, existing_records: dict[str, Record] = None) -> Record: + def run_single(self, config: Configuration, existing_records: Optional[dict[str, Record]] = None) -> Record: """Run the checks for a single repository target. Please use Analyzer.run if you want to run the analysis for a config parsed from @@ -312,7 +314,7 @@ def run_single(self, config: Configuration, existing_records: dict[str, Record] config : str The configuration for running Macaron. - existing_records : dict[str, Record] + existing_records : Optional[dict[str, Record]] The mapping of existing records that the analysis has run successfully. Returns diff --git a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py index 5b7d95d01..477a53537 100644 --- a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py +++ b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py @@ -36,91 +36,17 @@ def file_exists(path: str, file_name: str) -> bool: return False -def _find_parent_file_in(path: str, name: str) -> str: - """Return the path to the highest level file in a directory given its name. - - When only one instance of that file exists, this method will return the path - to that file. - - When multiple instances of that file exists, this method will ONLY return the path - to the highest level instance of that file. - - Parameters - ---------- - path : str - The path to the target dir. - name : str - The name of the file to search for. - - Returns - ------- - str - The path to the highest level file or empty if errors. - - Examples - -------- - .. code-block: bash - :caption: Given a dir as below - - a/ - ├── pom.xml - ├── b - │ └── pom.xml - └── c - └── d - └── pom.xml - └── e - └── pom.xml - - >>> _find_parent_file_in("a", "pom.xml") - 'a/pom.xml' - - >>> _find_parent_file_in("a/b", "pom.xml") - 'a/b/pom.xml' - - >>> _find_parent_file_in("a/b/c", "pom.xml") - '' - """ - # Search for all files in the path. - files_detected = glob.glob(os.path.join(path, "**", name), recursive=True) - if not files_detected: - logger.debug("Cannot find any %s in %s", name, path) - return "" - - # Return if there is only one instance of the file - # This is to avoid using os.path.commonpath on a single path, - # which would return that path instead of a parent dir. - if len(files_detected) == 1: - return files_detected.pop() - - # Get the path of the highest level file. - common_path = os.path.commonpath(files_detected) - parent_file = glob.glob(os.path.join(common_path, name)) - - # There cannot be two same instances of a file in the same dir. - if len(parent_file) > 1: - logger.critical("Find two instances of %s in %s", name, common_path) - return "" - - if not parent_file: - logger.debug("Cannot find the parent %s in the path %s", name, path) - return "" - - parent_file_path = parent_file.pop() - logger.debug("Found parent %s at %s", name, parent_file_path) - return parent_file_path - - class BaseBuildTool: - """This abstract class is used to implement Build Tools. - - Parameters - ---------- - name : str - The name of this build tool. - """ + """This abstract class is used to implement Build Tools.""" def __init__(self, name: str) -> None: + """Initialize instance. + + Parameters + ---------- + name : str + The name of this build tool. + """ self.name = name self.entry_conf: list[str] = [] self.build_configs: list[str] = [] @@ -193,13 +119,42 @@ class NoneBuildTool(BaseBuildTool): """This class can be used to initialize an empty build tool.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__(name="") def is_detected(self, repo_path: str) -> bool: + """Return True if this build tool is used in the target repo. + + Parameters + ---------- + repo_path : str + The path to the target repo. + + Returns + ------- + bool + True if this build tool is detected, else False. + """ return False def prepare_config_files(self, wrapper_path: str, build_dir: str) -> bool: + """Prepare the necessary wrapper files for running the build. + + This method will return False if there is any errors happened during operation. + + Parameters + ---------- + wrapper_path : str + The path where all necessary wrapper files are located. + build_dir : str + The path of the build dir. This is where all files are copied to. + + Returns + ------- + bool + True if succeed else False. + """ return False def load_defaults(self) -> None: - pass + """Load the default values from defaults.ini.""" diff --git a/src/macaron/slsa_analyzer/build_tool/gradle.py b/src/macaron/slsa_analyzer/build_tool/gradle.py index 5614f068f..32d5b6ede 100644 --- a/src/macaron/slsa_analyzer/build_tool/gradle.py +++ b/src/macaron/slsa_analyzer/build_tool/gradle.py @@ -20,6 +20,7 @@ class Gradle(BaseBuildTool): """This class contains the information of the Gradle build tool.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__(name="gradle") def load_defaults(self) -> None: @@ -40,6 +41,18 @@ def load_defaults(self) -> None: self.ci_deploy_kws[item] = defaults.get_list("builder.gradle.ci.deploy", item) def is_detected(self, repo_path: str) -> bool: + """Return True if this build tool is used in the target repo. + + Parameters + ---------- + repo_path : str + The path to the target repo. + + Returns + ------- + bool + True if this build tool is detected, else False. + """ gradle_config_files = self.build_configs + self.entry_conf for file in gradle_config_files: if file_exists(repo_path, file): diff --git a/src/macaron/slsa_analyzer/build_tool/maven.py b/src/macaron/slsa_analyzer/build_tool/maven.py index b306f08aa..f785baecf 100644 --- a/src/macaron/slsa_analyzer/build_tool/maven.py +++ b/src/macaron/slsa_analyzer/build_tool/maven.py @@ -20,6 +20,7 @@ class Maven(BaseBuildTool): """This class contains the information of the Maven build tool.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__(name="maven") def load_defaults(self) -> None: @@ -40,6 +41,18 @@ def load_defaults(self) -> None: self.ci_deploy_kws[item] = defaults.get_list("builder.maven.ci.deploy", item) def is_detected(self, repo_path: str) -> bool: + """Return True if this build tool is used in the target repo. + + Parameters + ---------- + repo_path : str + The path to the target repo. + + Returns + ------- + bool + True if this build tool is detected, else False. + """ maven_config_files = self.build_configs for file in maven_config_files: if file_exists(repo_path, file): diff --git a/src/macaron/slsa_analyzer/checks/base_check.py b/src/macaron/slsa_analyzer/checks/base_check.py index e5883168c..a2f462d21 100644 --- a/src/macaron/slsa_analyzer/checks/base_check.py +++ b/src/macaron/slsa_analyzer/checks/base_check.py @@ -5,6 +5,7 @@ import logging from abc import abstractmethod +from typing import Optional from macaron.slsa_analyzer.analyze_context import AnalyzeContext from macaron.slsa_analyzer.checks.check_result import CheckResult, CheckResultType, SkippedInfo, get_result_as_bool @@ -14,35 +15,36 @@ class BaseCheck: - """This abstract class is used to implement Checks in Macaron. - - Parameters - ---------- - check_id : str - The id of the check. - description : str - The description of the check. - depends_on : list[tuple(str, CheckResultType)] - The list of parent checks that this check depends on. - Each member of the list is a tuple of the parent's id and the status - of that parent check. - eval_reqs : list[ReqName] - The list of SLSA requirements that this check addresses. - result_on_skip : CheckResultType - The status for this check when it's skipped based on another check's result. - """ + """This abstract class is used to implement Checks in Macaron.""" + # The dictionary that contains the data of all SLSA requirements. SLSA_REQ_DATA = get_requirements_dict() - """The dictionary that contains the data of all SLSA requirements.""" def __init__( self, check_id: str = "", description: str = "", - depends_on: list[tuple[str, CheckResultType]] = None, - eval_reqs: list[ReqName] = None, + depends_on: Optional[list[tuple[str, CheckResultType]]] = None, + eval_reqs: Optional[list[ReqName]] = None, result_on_skip: CheckResultType = CheckResultType.SKIPPED, ) -> None: + """Initialize instance. + + Parameters + ---------- + check_id : str + The id of the check. + description : str + The description of the check. + depends_on : Optional[list[tuple(str, CheckResultType)]] + The list of parent checks that this check depends on. + Each member of the list is a tuple of the parent's id and the status + of that parent check. + eval_reqs : Optional[list[ReqName]] + The list of SLSA requirements that this check addresses. + result_on_skip : CheckResultType + The status for this check when it's skipped based on another check's result. + """ self.check_id = check_id self.description = description @@ -58,14 +60,14 @@ def __init__( self.result_on_skip = result_on_skip - def run(self, target: AnalyzeContext, skipped_info: SkippedInfo = None) -> CheckResult: + def run(self, target: AnalyzeContext, skipped_info: Optional[SkippedInfo] = None) -> CheckResult: """Run the check and return the results. Parameters ---------- target : AnalyzeContext The object containing processed data for the target repo. - skipped_info : SkippedInfo + skipped_info : Optional[SkippedInfo] Determine whether the check is skipped. Returns diff --git a/src/macaron/slsa_analyzer/checks/build_as_code_check.py b/src/macaron/slsa_analyzer/checks/build_as_code_check.py index 963950fde..fce124430 100644 --- a/src/macaron/slsa_analyzer/checks/build_as_code_check.py +++ b/src/macaron/slsa_analyzer/checks/build_as_code_check.py @@ -74,6 +74,20 @@ def _has_deploy_command(self, commands: list[list[str]], build_tool: BaseBuildTo return "" def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ # Get the build tool identified by the mcn_version_control_system_1, which we depend on. build_tool = ctx.dynamic_data["build_spec"].get("tool") ci_services = ctx.dynamic_data["ci_services"] diff --git a/src/macaron/slsa_analyzer/checks/build_script_check.py b/src/macaron/slsa_analyzer/checks/build_script_check.py index 4a9a1980d..4bc18701b 100644 --- a/src/macaron/slsa_analyzer/checks/build_script_check.py +++ b/src/macaron/slsa_analyzer/checks/build_script_check.py @@ -33,6 +33,20 @@ def __init__(self) -> None: ) def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ build_tool = ctx.dynamic_data["build_spec"].get("tool") # Check if a build tool is discovered for this repo. diff --git a/src/macaron/slsa_analyzer/checks/build_service_check.py b/src/macaron/slsa_analyzer/checks/build_service_check.py index fa35226e7..08d3ea6c9 100644 --- a/src/macaron/slsa_analyzer/checks/build_service_check.py +++ b/src/macaron/slsa_analyzer/checks/build_service_check.py @@ -65,6 +65,20 @@ def _has_build_command(self, commands: list[list[str]], build_tool: BaseBuildToo return "" def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ build_tool = ctx.dynamic_data["build_spec"].get("tool") ci_services = ctx.dynamic_data["ci_services"] diff --git a/src/macaron/slsa_analyzer/checks/check_result.py b/src/macaron/slsa_analyzer/checks/check_result.py index ee52c781e..88ea7b269 100644 --- a/src/macaron/slsa_analyzer/checks/check_result.py +++ b/src/macaron/slsa_analyzer/checks/check_result.py @@ -40,7 +40,7 @@ class CheckResult(TypedDict): class SkippedInfo(TypedDict): """This class stores the information about a skipped check.""" - id: str + check_id: str suppress_comment: str diff --git a/src/macaron/slsa_analyzer/checks/policy_check.py b/src/macaron/slsa_analyzer/checks/policy_check.py index 824ba3f2e..5890a53f3 100644 --- a/src/macaron/slsa_analyzer/checks/policy_check.py +++ b/src/macaron/slsa_analyzer/checks/policy_check.py @@ -22,6 +22,7 @@ class PolicyCheck(BaseCheck): """This check compares a SLSA provenance with a given policy and checks whether they match.""" def __init__(self) -> None: + """Initialize instance.""" check_id = "mcn_policy_check_1" description = "Check whether the SLSA provenance for the produced artifact conforms to the policy." depends_on: list[tuple[str, CheckResultType]] = [("mcn_provenance_level_three_1", CheckResultType.PASSED)] @@ -35,6 +36,20 @@ def __init__(self) -> None: ) def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ policy = ctx.dynamic_data["policy"] if not policy: check_result["justification"].append("Could not verify policy against the provenance.") diff --git a/src/macaron/slsa_analyzer/checks/provenance_available_check.py b/src/macaron/slsa_analyzer/checks/provenance_available_check.py index 1c41e7927..81c3bda6c 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_available_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_available_check.py @@ -42,6 +42,7 @@ class ProvenanceAvailableCheck(BaseCheck): """This Check checks whether the target repo has intoto provenance.""" def __init__(self) -> None: + """Initialize instance.""" check_id = "mcn_provenance_available_1" description = "Check whether the target has intoto provenance." depends_on: list[tuple[str, CheckResultType]] = [] @@ -54,6 +55,20 @@ def __init__(self) -> None: super().__init__(check_id=check_id, description=description, depends_on=depends_on, eval_reqs=eval_reqs) def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ ci_services = ctx.dynamic_data["ci_services"] for ci_info in ci_services: ci_service = ci_info["service"] diff --git a/src/macaron/slsa_analyzer/checks/provenance_l3_check.py b/src/macaron/slsa_analyzer/checks/provenance_l3_check.py index e392e1d61..102309476 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_l3_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_l3_check.py @@ -31,6 +31,7 @@ class ProvenanceL3Check(BaseCheck): """This Check checks whether the target repo has SLSA provenance level 3.""" def __init__(self) -> None: + """Initialize instance.""" check_id = "mcn_provenance_level_three_1" description = "Check whether the target has SLSA provenance level 3." depends_on: list[tuple[str, CheckResultType]] = [("mcn_provenance_available_1", CheckResultType.PASSED)] @@ -99,7 +100,7 @@ def _verify_slsa(self, macaron_path: str, temp_path: str, prov_asset: dict, asse except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: logger.error(error) errors.append(error.output.decode("utf-8")) - except (FileNotFoundError, OSError) as error: + except OSError as error: logger.error(error) errors.append(str(error)) @@ -113,7 +114,7 @@ def _verify_slsa(self, macaron_path: str, temp_path: str, prov_asset: dict, asse log_file.write(f"SLSA verifier output for cmd: {' '.join(cmd)}\n") log_file.writelines(errors) log_file.write("--------------------------------\n") - except (FileNotFoundError, OSError) as error: + except OSError as error: logger.error(error) return feedback @@ -185,6 +186,20 @@ def _find_asset( return None def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ # TODO: During verification, we need to fetch the workflow and verify that it's not # using self-hosted runners, custom containers or services, etc. all_feedback = [] diff --git a/src/macaron/slsa_analyzer/checks/trusted_builder_l3_check.py b/src/macaron/slsa_analyzer/checks/trusted_builder_l3_check.py index 2fac9321f..613afc597 100644 --- a/src/macaron/slsa_analyzer/checks/trusted_builder_l3_check.py +++ b/src/macaron/slsa_analyzer/checks/trusted_builder_l3_check.py @@ -23,6 +23,7 @@ class TrustedBuilderL3Check(BaseCheck): """This Check checks whether the target repo uses level 3 builders.""" def __init__(self) -> None: + """Initialize instance.""" check_id = "mcn_trusted_builder_level_three_1" description = "Check whether the target uses a trusted SLSA level 3 builder." depends_on: list[tuple[str, CheckResultType]] = [("mcn_version_control_system_1", CheckResultType.PASSED)] @@ -54,6 +55,20 @@ def __init__(self) -> None: ) def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ # TODO: During verification, we need to fetch the workflow and verify that it's not # using self-hosted runners, custom containers or services, etc. found_builder = False diff --git a/src/macaron/slsa_analyzer/checks/vcs_check.py b/src/macaron/slsa_analyzer/checks/vcs_check.py index ffc47675e..511a8a85f 100644 --- a/src/macaron/slsa_analyzer/checks/vcs_check.py +++ b/src/macaron/slsa_analyzer/checks/vcs_check.py @@ -15,6 +15,7 @@ class VCSCheck(BaseCheck): """This Check checks whether the target repo uses a version control system.""" def __init__(self) -> None: + """Initialize instance.""" check_id = "mcn_version_control_system_1" description = "Check whether the target repo uses a version control system." depends_on: list[tuple[str, CheckResultType]] = [] @@ -22,6 +23,20 @@ def __init__(self) -> None: super().__init__(check_id=check_id, description=description, depends_on=depends_on, eval_reqs=eval_reqs) def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ # TODO: refactor and use the git_service and its API client to create # the hyperlink tag to allow validation. if not ctx.git_obj: diff --git a/src/macaron/slsa_analyzer/ci_service/base_ci_service.py b/src/macaron/slsa_analyzer/ci_service/base_ci_service.py index c528acb98..82a5c1131 100644 --- a/src/macaron/slsa_analyzer/ci_service/base_ci_service.py +++ b/src/macaron/slsa_analyzer/ci_service/base_ci_service.py @@ -6,7 +6,7 @@ import logging import os from abc import abstractmethod -from typing import Iterable +from collections.abc import Iterable from macaron.code_analyzer.call_graph import BaseNode, CallGraph from macaron.parsers.bashparser import BashCommands @@ -16,15 +16,16 @@ class BaseCIService: - """This abstract class is used to implement CI services. - - Parameters - ---------- - name : str - The name of the CI service. - """ + """This abstract class is used to implement CI services.""" def __init__(self, name: str) -> None: + """Initialize instance. + + Parameters + ---------- + name : str + The name of the CI service. + """ self.name = name self.entry_conf: list[str] = [] # The file or dir that determines a CI service. self.api_client: BaseAPIClient = BaseAPIClient() @@ -83,7 +84,7 @@ def is_detected(self, repo_path: str) -> bool: return exists @abstractmethod - def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGraph: + def build_call_graph(self, repo_path: str, macaron_path: str = "") -> CallGraph: """Build the call Graph for this CI service. Parameters @@ -101,7 +102,7 @@ def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGrap raise NotImplementedError @abstractmethod - def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = None) -> Iterable[BashCommands]: + def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = "") -> Iterable[BashCommands]: """Parse configurations to extract the bash scripts triggered by the CI service. Parameters @@ -187,24 +188,88 @@ class NoneCIService(BaseCIService): """This class can be used to initialize an empty CI service.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__(name="") def get_workflows(self, repo_path: str) -> list: + """Get all workflows in a repository. + + Parameters + ---------- + repo_path : str + The path to the repository. + + Returns + ------- + list + The list of workflow files in this repository. + """ return [] def load_defaults(self) -> None: - pass + """Load the default values from defaults.ini.""" def set_api_client(self) -> None: - pass + """Set the API client using the personal access token.""" + + def build_call_graph(self, repo_path: str, macaron_path: str = "") -> CallGraph: + """Build the call Graph for this CI service. - def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGraph: + Parameters + ---------- + repo_path : str + The path to the repo. + macaron_path : str + Macaron's root path (optional). + + Returns + ------- + CallGraph : CallGraph + The call graph built for the CI. + """ return CallGraph(BaseNode(), "") - def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = None) -> Iterable[BashCommands]: + def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = "") -> Iterable[BashCommands]: + """Parse configurations to extract the bash scripts triggered by the CI service. + + Parameters + ---------- + callgraph : CallGraph + The call graph for this CI. + macaron_path : str + Macaron's root path (optional). + + Yields + ------ + BashCommands + The parsed bash script commands. + """ return [] def has_latest_run_passed( self, repo_full_name: str, branch_name: str, commit_sha: str, commit_date: str, workflow: str ) -> str: + """Get the latest run of a workflow in the repository. + + This workflow run must be based on the latest commit according to the commit sha in the + Analyze context. + + Parameters + ---------- + repo_full_name : str + The target repo's full name. + branch_name : str + The target branch. + commit_sha : str + The commit sha of the target repo. + commit_date : str + The commit date of the target repo. + workflow : str + The name of the workflow file (e.g `build.yml`). + + Returns + ------- + str + The feed back of the check, or empty if no passing workflow is found. + """ return "" diff --git a/src/macaron/slsa_analyzer/ci_service/circleci.py b/src/macaron/slsa_analyzer/ci_service/circleci.py index 913343e1c..491a3df76 100644 --- a/src/macaron/slsa_analyzer/ci_service/circleci.py +++ b/src/macaron/slsa_analyzer/ci_service/circleci.py @@ -3,7 +3,7 @@ """This module analyze Circle CI.""" -from typing import Iterable +from collections.abc import Iterable from macaron.code_analyzer.call_graph import BaseNode, CallGraph from macaron.config.defaults import defaults @@ -15,10 +15,23 @@ class CircleCI(BaseCIService): """This class implements CircleCI service.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__(name="circle_ci") self.entry_conf = [".circleci/config.yml"] def get_workflows(self, repo_path: str) -> list: + """Get all workflows in a repository. + + Parameters + ---------- + repo_path : str + The path to the repository. + + Returns + ------- + list + The list of workflow files in this repository. + """ return [] def load_defaults(self) -> None: @@ -29,15 +42,66 @@ def load_defaults(self) -> None: setattr(self, item, defaults.get_list("ci.circle_ci", item)) def set_api_client(self) -> None: - pass + """Set the API client using the personal access token.""" + + def build_call_graph(self, repo_path: str, macaron_path: str = "") -> CallGraph: + """Build the call Graph for this CI service. - def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGraph: + Parameters + ---------- + repo_path : str + The path to the repo. + macaron_path : str + Macaron's root path (optional). + + Returns + ------- + CallGraph : CallGraph + The call graph built for the CI. + """ return CallGraph(BaseNode(), "") - def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = None) -> Iterable[BashCommands]: + def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = "") -> Iterable[BashCommands]: + """Parse configurations to extract the bash scripts triggered by the CI service. + + Parameters + ---------- + callgraph : CallGraph + The call graph for this CI. + macaron_path : str + Macaron's root path (optional). + + Yields + ------ + BashCommands + The parsed bash script commands. + """ return [] def has_latest_run_passed( self, repo_full_name: str, branch_name: str, commit_sha: str, commit_date: str, workflow: str ) -> str: + """Get the latest run of a workflow in the repository. + + This workflow run must be based on the latest commit according to the commit sha in the + Analyze context. + + Parameters + ---------- + repo_full_name : str + The target repo's full name. + branch_name : str + The target branch. + commit_sha : str + The commit sha of the target repo. + commit_date : str + The commit date of the target repo. + workflow : str + The name of the workflow file (e.g `build.yml`). + + Returns + ------- + str + The feed back of the check, or empty if no passing workflow is found. + """ return "" diff --git a/src/macaron/slsa_analyzer/ci_service/github_actions.py b/src/macaron/slsa_analyzer/ci_service/github_actions.py index e34d5daa4..7ae85dee9 100644 --- a/src/macaron/slsa_analyzer/ci_service/github_actions.py +++ b/src/macaron/slsa_analyzer/ci_service/github_actions.py @@ -6,9 +6,9 @@ import glob import logging import os +from collections.abc import Iterable from datetime import datetime, timezone from enum import Enum -from typing import Iterable from macaron.code_analyzer.call_graph import BaseNode, CallGraph from macaron.config.defaults import defaults @@ -31,25 +31,26 @@ class GHWorkflowType(Enum): class GitHubNode(BaseNode): - """This class is used to create a call graph node for GitHub Actions. - - Parameters - ---------- - name : str - Name of the workflow (or URL for reusable and external workflows). - node_type : GHWorkflowType - The type of workflow. - source_path : str - The path of the workflow. - parsed_obj : dict - The parsed Actions workflow object. - caller_path : str - The path to the caller workflow. - """ + """This class is used to create a call graph node for GitHub Actions.""" def __init__( self, name: str, node_type: GHWorkflowType, source_path: str, parsed_obj: dict, caller_path: str ) -> None: + """Initialize instance. + + Parameters + ---------- + name : str + Name of the workflow (or URL for reusable and external workflows). + node_type : GHWorkflowType + The type of workflow. + source_path : str + The path of the workflow. + parsed_obj : dict + The parsed Actions workflow object. + caller_path : str + The path to the caller workflow. + """ super().__init__() self.name = name self.node_type: GHWorkflowType = node_type @@ -65,6 +66,7 @@ class GitHubActions(BaseCIService): """This class contains the spec of the GitHub Actions.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__(name="github_actions") self.personal_access_token = "" # nosec B105 self.api_client: GhAPIClient = get_default_gh_client("") @@ -81,18 +83,32 @@ def set_api_client(self) -> None: def load_defaults(self) -> None: """Load the default values from defaults.ini.""" if "ci.github_actions" in defaults: - setattr( + setattr( # noqa: B010 self, "query_page_threshold", defaults.getint("ci.github_actions", "query_page_threshold", fallback=10) ) - setattr(self, "max_items_num", defaults.getint("ci.github_actions", "max_items_num", fallback=100)) - setattr( + setattr( # noqa: B010 + self, "max_items_num", defaults.getint("ci.github_actions", "max_items_num", fallback=100) + ) + setattr( # noqa: B010 self, "entry_conf", defaults.get_list("ci.github_actions", "entry_conf", fallback=[".github/workflows"]) ) - setattr( + setattr( # noqa: B010 self, "max_workflow_persist", defaults.getint("ci.github_actions", "max_workflow_persist", fallback=90) ) def is_detected(self, repo_path: str) -> bool: + """Return True if this CI service is used in the target repo. + + Parameters + ---------- + repo_path : str + The path to the target repo. + + Returns + ------- + bool + True if this CI service is detected, else False. + """ # GitHub Actions need a special detection implementation. # We need to check if YAML files exist in the workflows dir. exists = False @@ -174,7 +190,7 @@ def build_call_graph_from_node(self, node: GitHubNode) -> None: ) ) - def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGraph: + def build_call_graph(self, repo_path: str, macaron_path: str = "") -> CallGraph: """Build the call Graph for GitHub Actions workflows. At the moment it does not analyze third-party workflows to include their callees. @@ -191,7 +207,7 @@ def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGrap CallGraph: CallGraph The call graph built for GitHub Actions. """ - if macaron_path is None: + if not macaron_path: macaron_path = global_config.macaron_path root = GitHubNode(name="", node_type=GHWorkflowType.NONE, source_path="", parsed_obj={}, caller_path="") @@ -223,7 +239,7 @@ def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGrap return gh_cg - def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = None) -> Iterable[BashCommands]: + def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = "") -> Iterable[BashCommands]: """Extract the bash scripts triggered by the CI service from parsing the configurations. Parameters @@ -238,7 +254,7 @@ def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = None) -> It BashCommands The parsed bash script commands. """ - if macaron_path is None: + if not macaron_path: macaron_path = global_config.macaron_path # Analyze GitHub Actions workflows. @@ -321,7 +337,7 @@ def has_latest_run_passed( logger.debug("Error while calculating the delta time of commit date: %s.", error) workflow_data = self.api_client.get_repo_workflow_data(repo_full_name, workflow) - if workflow_data == {}: + if not workflow_data: logger.error("Cannot find data of workflow %s.", workflow) return "" @@ -363,8 +379,8 @@ def has_latest_run_passed( # Skip this workflow when it's failing. try: - run_id = latest_run_data["id"] - html_url = latest_run_data["html_url"] + run_id: str = latest_run_data["id"] + html_url: str = latest_run_data["html_url"] if latest_run_data["conclusion"] != "success": logger.info("The workflow run for %s was unsuccessful. Skipping ....", workflow) return "" @@ -443,7 +459,7 @@ def search_for_workflow_run( for run in runs_data["workflow_runs"]: if run["workflow_id"] == workflow_id and run["head_sha"] == commit_sha: logger.info("Found workflow run of %s in page %s.", workflow_id, query_page) - return run + return dict(run) logger.info("Didn't find any target run of %s on page %s.", workflow_id, query_page) if len(runs_data["workflow_runs"]) < self.max_items_num: diff --git a/src/macaron/slsa_analyzer/ci_service/gitlab_ci.py b/src/macaron/slsa_analyzer/ci_service/gitlab_ci.py index bc1b0827d..90ec7e68b 100644 --- a/src/macaron/slsa_analyzer/ci_service/gitlab_ci.py +++ b/src/macaron/slsa_analyzer/ci_service/gitlab_ci.py @@ -3,7 +3,7 @@ """This module analyzes GitLab CI.""" -from typing import Iterable +from collections.abc import Iterable from macaron.code_analyzer.call_graph import BaseNode, CallGraph from macaron.config.defaults import defaults @@ -15,10 +15,23 @@ class GitLabCI(BaseCIService): """This class implements GitLab CI service.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__(name="gitlab_ci") self.entry_conf = [".gitlab-ci.yml", ".gitlab-ci.yaml"] def get_workflows(self, repo_path: str) -> list: + """Get all workflows in a repository. + + Parameters + ---------- + repo_path : str + The path to the repository. + + Returns + ------- + list + The list of workflow files in this repository. + """ return [] def load_defaults(self) -> None: @@ -29,15 +42,66 @@ def load_defaults(self) -> None: setattr(self, item, defaults.get_list("ci.gitlab_ci", item)) def set_api_client(self) -> None: - pass + """Set the API client using the personal access token.""" + + def build_call_graph(self, repo_path: str, macaron_path: str = "") -> CallGraph: + """Build the call Graph for this CI service. - def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGraph: + Parameters + ---------- + repo_path : str + The path to the repo. + macaron_path : str + Macaron's root path (optional). + + Returns + ------- + CallGraph : CallGraph + The call graph built for the CI. + """ return CallGraph(BaseNode(), "") - def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = None) -> Iterable[BashCommands]: + def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = "") -> Iterable[BashCommands]: + """Parse configurations to extract the bash scripts triggered by the CI service. + + Parameters + ---------- + callgraph : CallGraph + The call graph for this CI. + macaron_path : str + Macaron's root path (optional). + + Yields + ------ + BashCommands + The parsed bash script commands. + """ return [] def has_latest_run_passed( self, repo_full_name: str, branch_name: str, commit_sha: str, commit_date: str, workflow: str ) -> str: + """Get the latest run of a workflow in the repository. + + This workflow run must be based on the latest commit according to the commit sha in the + Analyze context. + + Parameters + ---------- + repo_full_name : str + The target repo's full name. + branch_name : str + The target branch. + commit_sha : str + The commit sha of the target repo. + commit_date : str + The commit date of the target repo. + workflow : str + The name of the workflow file (e.g `build.yml`). + + Returns + ------- + str + The feed back of the check, or empty if no passing workflow is found. + """ return "" diff --git a/src/macaron/slsa_analyzer/ci_service/jenkins.py b/src/macaron/slsa_analyzer/ci_service/jenkins.py index 3d5da7a32..3a2076843 100644 --- a/src/macaron/slsa_analyzer/ci_service/jenkins.py +++ b/src/macaron/slsa_analyzer/ci_service/jenkins.py @@ -3,7 +3,7 @@ """This module analyzes Jenkins CI.""" -from typing import Iterable +from collections.abc import Iterable from macaron.code_analyzer.call_graph import BaseNode, CallGraph from macaron.config.defaults import defaults @@ -15,10 +15,23 @@ class Jenkins(BaseCIService): """This class implements Jenkins CI service.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__(name="jenkins") self.entry_conf = ["Jenkinsfile"] def get_workflows(self, repo_path: str) -> list: + """Get all workflows in a repository. + + Parameters + ---------- + repo_path : str + The path to the repository. + + Returns + ------- + list + The list of workflow files in this repository. + """ return [] def load_defaults(self) -> None: @@ -29,15 +42,66 @@ def load_defaults(self) -> None: setattr(self, item, defaults.get_list("ci.jenkins", item)) def set_api_client(self) -> None: - pass + """Set the API client using the personal access token.""" + + def build_call_graph(self, repo_path: str, macaron_path: str = "") -> CallGraph: + """Build the call Graph for this CI service. - def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGraph: + Parameters + ---------- + repo_path : str + The path to the repo. + macaron_path : str + Macaron's root path (optional). + + Returns + ------- + CallGraph : CallGraph + The call graph built for the CI. + """ return CallGraph(BaseNode(), "") - def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = None) -> Iterable[BashCommands]: + def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = "") -> Iterable[BashCommands]: + """Parse configurations to extract the bash scripts triggered by the CI service. + + Parameters + ---------- + callgraph : CallGraph + The call graph for this CI. + macaron_path : str + Macaron's root path (optional). + + Yields + ------ + BashCommands + The parsed bash script commands. + """ return [] def has_latest_run_passed( self, repo_full_name: str, branch_name: str, commit_sha: str, commit_date: str, workflow: str ) -> str: + """Get the latest run of a workflow in the repository. + + This workflow run must be based on the latest commit according to the commit sha in the + Analyze context. + + Parameters + ---------- + repo_full_name : str + The target repo's full name. + branch_name : str + The target branch. + commit_sha : str + The commit sha of the target repo. + commit_date : str + The commit date of the target repo. + workflow : str + The name of the workflow file (e.g `build.yml`). + + Returns + ------- + str + The feed back of the check, or empty if no passing workflow is found. + """ return "" diff --git a/src/macaron/slsa_analyzer/ci_service/travis.py b/src/macaron/slsa_analyzer/ci_service/travis.py index 197ce32a3..b953b9978 100644 --- a/src/macaron/slsa_analyzer/ci_service/travis.py +++ b/src/macaron/slsa_analyzer/ci_service/travis.py @@ -3,7 +3,7 @@ """This module analyzes Travis CI.""" -from typing import Iterable +from collections.abc import Iterable from macaron.code_analyzer.call_graph import BaseNode, CallGraph from macaron.config.defaults import defaults @@ -15,10 +15,23 @@ class Travis(BaseCIService): """This class implements Travis CI service.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__(name="travis_ci") self.entry_conf = [".travis.yml", ".travis.yaml"] def get_workflows(self, repo_path: str) -> list: + """Get all workflows in a repository. + + Parameters + ---------- + repo_path : str + The path to the repository. + + Returns + ------- + list + The list of workflow files in this repository. + """ return [] def load_defaults(self) -> None: @@ -29,15 +42,66 @@ def load_defaults(self) -> None: setattr(self, item, defaults.get_list("ci.travis_ci", item)) def set_api_client(self) -> None: - pass + """Set the API client using the personal access token.""" + + def build_call_graph(self, repo_path: str, macaron_path: str = "") -> CallGraph: + """Build the call Graph for this CI service. - def build_call_graph(self, repo_path: str, macaron_path: str = None) -> CallGraph: + Parameters + ---------- + repo_path : str + The path to the repo. + macaron_path : str + Macaron's root path (optional). + + Returns + ------- + CallGraph : CallGraph + The call graph built for the CI. + """ return CallGraph(BaseNode(), "") - def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = None) -> Iterable[BashCommands]: + def extract_all_bash(self, callgraph: CallGraph, macaron_path: str = "") -> Iterable[BashCommands]: + """Parse configurations to extract the bash scripts triggered by the CI service. + + Parameters + ---------- + callgraph : CallGraph + The call graph for this CI. + macaron_path : str + Macaron's root path (optional). + + Yields + ------ + BashCommands + The parsed bash script commands. + """ return [] def has_latest_run_passed( self, repo_full_name: str, branch_name: str, commit_sha: str, commit_date: str, workflow: str ) -> str: + """Get the latest run of a workflow in the repository. + + This workflow run must be based on the latest commit according to the commit sha in the + Analyze context. + + Parameters + ---------- + repo_full_name : str + The target repo's full name. + branch_name : str + The target branch. + commit_sha : str + The commit sha of the target repo. + commit_date : str + The commit date of the target repo. + workflow : str + The name of the workflow file (e.g `build.yml`). + + Returns + ------- + str + The feed back of the check, or empty if no passing workflow is found. + """ return "" diff --git a/src/macaron/slsa_analyzer/git_service/base_git_service.py b/src/macaron/slsa_analyzer/git_service/base_git_service.py index 2d076e0fe..a3595ad0d 100644 --- a/src/macaron/slsa_analyzer/git_service/base_git_service.py +++ b/src/macaron/slsa_analyzer/git_service/base_git_service.py @@ -7,15 +7,16 @@ class BaseGitService: - """This abstract class is used to implement git services. - - Parameters - ---------- - name : str - The name of the git service. - """ + """This abstract class is used to implement git services.""" def __init__(self, name: str) -> None: + """Initialize instance. + + Parameters + ---------- + name : str + The name of the git service. + """ self.name = name @abstractmethod @@ -60,13 +61,38 @@ class NoneGitService(BaseGitService): """This class can be used to initialize an empty git service.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__("") def load_defaults(self) -> None: - pass + """Load the default values from defaults.ini.""" def is_detected(self, url: str) -> bool: + """Return True if the remote repo is using this git service. + + Parameters + ---------- + url : str + The url of the remote repo. + + Returns + ------- + bool + True if this git service is detected else False. + """ return False def can_clone_remote_repo(self, url: str) -> bool: + """Return True if the remote repository can be cloned. + + Parameters + ---------- + url : str + The remote url. + + Returns + ------- + bool + True if the repo can be cloned, else False. + """ return False diff --git a/src/macaron/slsa_analyzer/git_service/bitbucket.py b/src/macaron/slsa_analyzer/git_service/bitbucket.py index 58c8a2640..907573759 100644 --- a/src/macaron/slsa_analyzer/git_service/bitbucket.py +++ b/src/macaron/slsa_analyzer/git_service/bitbucket.py @@ -3,23 +3,53 @@ """This module contains the spec for the BitBucket service.""" +import logging + from macaron.slsa_analyzer import git_url from macaron.slsa_analyzer.git_service.base_git_service import BaseGitService +logger: logging.Logger = logging.getLogger(__name__) + class BitBucket(BaseGitService): """This class contains the spec of the BitBucket service.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__("bitbucket") def load_defaults(self) -> None: - pass + """Load the default values from defaults.ini.""" def can_clone_remote_repo(self, url: str) -> bool: - pass + """Return True if the remote repository can be cloned. + + Parameters + ---------- + url : str + The remote url. + + Returns + ------- + bool + True if the repo can be cloned, else False. + """ + logger.info("Cloning BitBucket repositories is not supported yet. Please clone the repository manually.") + return False def is_detected(self, url: str) -> bool: + """Return True if the remote repo is using this git service. + + Parameters + ---------- + url : str + The url of the remote repo. + + Returns + ------- + bool + True if this git service is detected else False. + """ parsed_url = git_url.parse_remote_url(url) if not parsed_url or self.name not in parsed_url.netloc: return False diff --git a/src/macaron/slsa_analyzer/git_service/github.py b/src/macaron/slsa_analyzer/git_service/github.py index 400e5d448..aa4974592 100644 --- a/src/macaron/slsa_analyzer/git_service/github.py +++ b/src/macaron/slsa_analyzer/git_service/github.py @@ -13,11 +13,12 @@ class GitHub(BaseGitService): """This class contains the spec of the GitHub service.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__("github") self._api_client: GhAPIClient = None # type: ignore def load_defaults(self) -> None: - pass + """Load the default values from defaults.ini.""" @property def api_client(self) -> GhAPIClient: @@ -31,6 +32,18 @@ def api_client(self) -> GhAPIClient: return self._api_client def can_clone_remote_repo(self, url: str) -> bool: + """Return True if the remote repository can be cloned. + + Parameters + ---------- + url : str + The remote url. + + Returns + ------- + bool + True if the repo can be cloned, else False. + """ remote_url = git_url.get_remote_vcs_url(url) full_name = git_url.get_repo_full_name_from_url(remote_url) if not self.api_client.get_repo_data(full_name): @@ -39,6 +52,18 @@ def can_clone_remote_repo(self, url: str) -> bool: return True def is_detected(self, url: str) -> bool: + """Return True if the remote repo is using this git service. + + Parameters + ---------- + url : str + The url of the remote repo. + + Returns + ------- + bool + True if this git service is detected else False. + """ parsed_url = git_url.parse_remote_url(url) if not parsed_url or self.name not in parsed_url.netloc: return False diff --git a/src/macaron/slsa_analyzer/git_service/gitlab.py b/src/macaron/slsa_analyzer/git_service/gitlab.py index d582ce08a..0501c2b6e 100644 --- a/src/macaron/slsa_analyzer/git_service/gitlab.py +++ b/src/macaron/slsa_analyzer/git_service/gitlab.py @@ -11,15 +11,40 @@ class GitLab(BaseGitService): """This class contains the spec of the GitLab service.""" def __init__(self) -> None: + """Initialize instance.""" super().__init__("gitlab") def load_defaults(self) -> None: - pass + """Load the default values from defaults.ini.""" def can_clone_remote_repo(self, url: str) -> bool: - pass + """Return True if the remote repository can be cloned. + + Parameters + ---------- + url : str + The remote url. + + Returns + ------- + bool + True if the repo can be cloned, else False. + """ + return False def is_detected(self, url: str) -> bool: + """Return True if the remote repo is using this git service. + + Parameters + ---------- + url : str + The url of the remote repo. + + Returns + ------- + bool + True if this git service is detected else False. + """ parsed_url = git_url.parse_remote_url(url) if not parsed_url or self.name not in parsed_url.netloc: return False diff --git a/src/macaron/slsa_analyzer/git_url.py b/src/macaron/slsa_analyzer/git_url.py index 08a65c3aa..656cb183d 100644 --- a/src/macaron/slsa_analyzer/git_url.py +++ b/src/macaron/slsa_analyzer/git_url.py @@ -9,6 +9,7 @@ import re import string import urllib.parse +from typing import Optional from git import GitCommandError from git.objects import Commit @@ -61,7 +62,7 @@ def reset_git_repo(git_obj: Git, stash: bool = True, index: bool = True, working return False -def check_out_repo_target(git_obj: Git, branch_name: str = None, digest: str = None) -> bool: +def check_out_repo_target(git_obj: Git, branch_name: str = "", digest: str = "") -> bool: """Checkout the branch and commit specified by the user. If no branch name is provided, this method will checkout the default branch @@ -195,7 +196,7 @@ def commit_exists(git_obj: Git, digest: str) -> bool: bool """ try: - return git_obj.repo.is_ancestor(digest, "HEAD") + return bool(git_obj.repo.is_ancestor(digest, "HEAD")) except GitCommandError as error: # The exception could be raised because the digest does not exist in the history of the branch # or the digest is not a valid digest. @@ -255,7 +256,7 @@ def get_default_branch(git_obj: Git) -> str: # This command will return origin/. # It can also work after we checkout a specific commit making HEAD into a detached state. # This is suitable for running multiple times on a repo. - default_branch_full = git_obj.repo.git.rev_parse("--abbrev-ref", "origin/HEAD") + default_branch_full: str = git_obj.repo.git.rev_parse("--abbrev-ref", "origin/HEAD") return default_branch_full[7:] except GitCommandError as error: logger.error("Error when getting default branch. Error: %s", error) @@ -313,7 +314,7 @@ def clone_remote_repo(clone_dir: str, url: str) -> Repo | None: "The clone dir %s is empty. It has been deleted for cloning the repo.", clone_dir, ) - except (FileNotFoundError, OSError): + except OSError: logger.info("The clone dir %s is not empty. No cloning is proceeded.", clone_dir) return None @@ -464,7 +465,7 @@ def get_remote_vcs_url(url: str, clean_up: bool = True) -> str: return url_as_str -def parse_remote_url(url: str, git_hosts: list = None) -> urllib.parse.ParseResult | None: +def parse_remote_url(url: str, git_hosts: Optional[list] = None) -> urllib.parse.ParseResult | None: """Verify if the given repository path is a valid vcs. This method converts the url to a ``https://`` url and return a @@ -475,7 +476,7 @@ def parse_remote_url(url: str, git_hosts: list = None) -> urllib.parse.ParseResu ---------- url: str The path of the repository to check. - git_hosts: list + git_hosts: Optional[list] The list of allowed network locations (default: None). Returns diff --git a/src/macaron/slsa_analyzer/registry.py b/src/macaron/slsa_analyzer/registry.py index 7b21642a0..d65cdb42d 100644 --- a/src/macaron/slsa_analyzer/registry.py +++ b/src/macaron/slsa_analyzer/registry.py @@ -172,7 +172,7 @@ def _validate_check(check: Any) -> bool: return False if check_file_abs_path: - if not isinstance(check.result_on_skip, CheckResultType): + if not (hasattr(check, "result_on_skip") and isinstance(check.result_on_skip, CheckResultType)): logger.error("The status_on_skipped in the Check at %s is invalid.", str(check.check_id)) return False @@ -556,7 +556,7 @@ def _should_skip_check(check: BaseCheck, results: dict[str, CheckResult]) -> Ski f"Check {check.check_id} is set to {check.result_on_skip.value} " f"because {parent_id} {got_status.value}." ) - skipped_info = SkippedInfo(id=check.check_id, suppress_comment=suppress_comment) + skipped_info = SkippedInfo(check_id=check.check_id, suppress_comment=suppress_comment) return skipped_info return None diff --git a/src/macaron/slsa_analyzer/runner.py b/src/macaron/slsa_analyzer/runner.py index 26c71ba6d..232d7bf50 100644 --- a/src/macaron/slsa_analyzer/runner.py +++ b/src/macaron/slsa_analyzer/runner.py @@ -16,18 +16,19 @@ # pylint: disable=too-few-public-methods class Runner: - """The Runner runs a Check in its own thread and returns the Check results. - - Parameters - ---------- - register - The Registry that initialized this Runner. - i : str - The id of this Runner instance - """ + """The Runner runs a Check in its own thread and returns the Check results.""" # We use Any to prevent circular dependency def __init__(self, registry: Any, i: int) -> None: + """Initialize instance. + + Parameters + ---------- + register + The Registry that initialized this Runner. + i : str + The id of this Runner instance + """ self.registry = registry self.runner_id = i @@ -57,9 +58,9 @@ def run( skip_info = None if skipped_checks: - if check.check_id in [skip["id"] for skip in skipped_checks]: + if check.check_id in [skip["check_id"] for skip in skipped_checks]: # Get the skip info from the list. - skip_info = [skip for skip in skipped_checks if skip["id"] == check.check_id][0] + skip_info = [skip for skip in skipped_checks if skip["check_id"] == check.check_id][0] check_result = check.run(target, skip_info) diff --git a/src/macaron/slsa_analyzer/slsa_req.py b/src/macaron/slsa_analyzer/slsa_req.py index 46fe54c60..916cd50c1 100644 --- a/src/macaron/slsa_analyzer/slsa_req.py +++ b/src/macaron/slsa_analyzer/slsa_req.py @@ -315,21 +315,22 @@ class Category(Enum): class SLSAReq: - """This class represents a SLSA requirement (e.g Version Controlled). - - Parameters - ---------- - name : str - The name of the SLSA requirement. - desc : str - The description of the SLSA requirement. - category : Category - The category of the SLSA requirement. - req_level : SLSALevels - The SLSA level that this requirement belongs to. - """ + """This class represents a SLSA requirement (e.g Version Controlled).""" def __init__(self, name: str, desc: str, category: Category, req_level: SLSALevels): + """Initialize instance. + + Parameters + ---------- + name : str + The name of the SLSA requirement. + desc : str + The description of the SLSA requirement. + category : Category + The category of the SLSA requirement. + req_level : SLSALevels + The SLSA level that this requirement belongs to. + """ self.name = name self.desc = desc self.category = category diff --git a/src/macaron/slsa_analyzer/specs/inferred_provenance.py b/src/macaron/slsa_analyzer/specs/inferred_provenance.py index 68b2b9445..f0e9b5869 100644 --- a/src/macaron/slsa_analyzer/specs/inferred_provenance.py +++ b/src/macaron/slsa_analyzer/specs/inferred_provenance.py @@ -8,6 +8,7 @@ class Provenance: """This class implements the inferred SLSA provenance.""" def __init__(self) -> None: + """Initialize instance.""" self.payload = { "_type": "https://in-toto.io/Statement/v0.1", "subject": [], diff --git a/src/macaron/util.py b/src/macaron/util.py index c7e263355..49b2a15ad 100644 --- a/src/macaron/util.py +++ b/src/macaron/util.py @@ -13,6 +13,8 @@ import requests from requests.models import Response +from macaron.config.defaults import defaults + logger: logging.Logger = logging.getLogger(__name__) @@ -34,7 +36,7 @@ def send_get_http(url: str, headers: dict) -> dict: The response's json data or an empty dict if there is an error. """ logger.debug("GET - %s", url) - response = requests.get(url=url, headers=headers) + response = requests.get(url=url, headers=headers, timeout=defaults.getint("requests", "timeout", fallback=10)) while response.status_code != 200: logger.error( "Receiving error code %s from server. Message: %s.", @@ -45,9 +47,9 @@ def send_get_http(url: str, headers: dict) -> dict: check_rate_limit(response) else: return {} - response = requests.get(url=url, headers=headers) + response = requests.get(url=url, headers=headers, timeout=defaults.getint("requests", "timeout", fallback=10)) - return response.json() + return dict(response.json()) def send_get_http_raw(url: str, headers: dict) -> Response | None: @@ -68,7 +70,7 @@ def send_get_http_raw(url: str, headers: dict) -> Response | None: The response object or None if there is an error. """ logger.debug("GET - %s", url) - response = requests.get(url=url, headers=headers) + response = requests.get(url=url, headers=headers, timeout=defaults.getint("requests", "timeout", fallback=10)) while response.status_code != 200: logger.error( "Receiving error code %s from server. Message: %s.", @@ -79,7 +81,7 @@ def send_get_http_raw(url: str, headers: dict) -> Response | None: check_rate_limit(response) else: return None - response = requests.get(url=url, headers=headers) + response = requests.get(url=url, headers=headers, timeout=defaults.getint("requests", "timeout", fallback=10)) return response @@ -153,7 +155,7 @@ def download_github_build_log(url: str, headers: dict) -> str: The content of the downloaded build log or empty if error. """ logger.debug("Downloading content at link %s", url) - response = requests.get(url=url, headers=headers) + response = requests.get(url=url, headers=headers, timeout=defaults.getint("requests", "timeout", fallback=10)) return response.content.decode("utf-8") diff --git a/tests/dependency_analyzer/compare_dependencies.py b/tests/dependency_analyzer/compare_dependencies.py index f31f1a20c..91670180b 100755 --- a/tests/dependency_analyzer/compare_dependencies.py +++ b/tests/dependency_analyzer/compare_dependencies.py @@ -4,8 +4,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """ -This module checks the dependency analysis -results against the expected outputs. +This module checks the dependency analysis results against the expected outputs. """ import json @@ -27,8 +26,8 @@ def main() -> None: # Iterate through the elements to provide useful debug info. # We could use deepdiff library, but let's avoid adding a third-party dependency. - result_sorted = sorted(result, key=lambda item: item["id"]) - expected_sorted = sorted(expected, key=lambda item: item["id"]) + result_sorted = sorted(result, key=lambda item: str(item["id"])) + expected_sorted = sorted(expected, key=lambda item: str(item["id"])) if len(result_sorted) < len(expected_sorted): for dep in expected_sorted[len(result_sorted) :]: diff --git a/tests/dependency_analyzer/configurations/jackson_databind_config.yaml b/tests/dependency_analyzer/configurations/jackson_databind_config.yaml index 7fbb18e22..5b286151b 100644 --- a/tests/dependency_analyzer/configurations/jackson_databind_config.yaml +++ b/tests/dependency_analyzer/configurations/jackson_databind_config.yaml @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "jackson-databind" - branch: "2.14" - digest: "6e9193f069d1cf5e9590afeaeb2907a9e43fb143" - path: "https://github.com/FasterXML/jackson-databind" + id: jackson-databind + branch: '2.14' + digest: 6e9193f069d1cf5e9590afeaeb2907a9e43fb143 + path: https://github.com/FasterXML/jackson-databind diff --git a/tests/dependency_analyzer/configurations/maven_config.yaml b/tests/dependency_analyzer/configurations/maven_config.yaml index b60e6aa60..19577c482 100644 --- a/tests/dependency_analyzer/configurations/maven_config.yaml +++ b/tests/dependency_analyzer/configurations/maven_config.yaml @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "apache/maven" - branch: "master" - digest: "6767f2500f1d005924ccff27f04350c253858a84" - path: "https://github.com/apache/maven.git" + id: apache/maven + branch: master + digest: 6767f2500f1d005924ccff27f04350c253858a84 + path: https://github.com/apache/maven.git diff --git a/tests/dependency_analyzer/configurations/micronaut_core_config.yaml b/tests/dependency_analyzer/configurations/micronaut_core_config.yaml index 18252915e..d8504dcfe 100644 --- a/tests/dependency_analyzer/configurations/micronaut_core_config.yaml +++ b/tests/dependency_analyzer/configurations/micronaut_core_config.yaml @@ -2,18 +2,18 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "micronaut-core" - branch: "3.5.x" - digest: "bf3f5e8f33b84160276acaa2b4f977e64730b8cd" - path: "https://github.com/micronaut-projects/micronaut-core" + id: micronaut-core + branch: 3.5.x + digest: bf3f5e8f33b84160276acaa2b4f977e64730b8cd + path: https://github.com/micronaut-projects/micronaut-core dependencies: - - id: "slf4j" - branch: "" - digest: "" - path: "https://github.com/qos-ch/slf4j.git" +- id: slf4j + branch: '' + digest: '' + path: https://github.com/qos-ch/slf4j.git - - id: "caffeine" - branch: "" - digest: "" - path: "https://github.com/ben-manes/caffeine.git" +- id: caffeine + branch: '' + digest: '' + path: https://github.com/ben-manes/caffeine.git diff --git a/tests/dependency_analyzer/configurations/valid_has_deps.yaml b/tests/dependency_analyzer/configurations/valid_has_deps.yaml index 2d40eafa0..33d5dc274 100644 --- a/tests/dependency_analyzer/configurations/valid_has_deps.yaml +++ b/tests/dependency_analyzer/configurations/valid_has_deps.yaml @@ -2,17 +2,17 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "id" - path: "https://github.com/owner/name.git" - branch: "master" - digest: "aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7" + id: id + path: https://github.com/owner/name.git + branch: master + digest: aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7 dependencies: - - id: "id" - path: "https://github.com/owner/name.git" - branch: "master" - digest: "aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7" - - id: "id" - path: "https://github.com/owner/name_2.git" - branch: "master" - digest: "aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7" +- id: id + path: https://github.com/owner/name.git + branch: master + digest: aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7 +- id: id + path: https://github.com/owner/name_2.git + branch: master + digest: aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7 diff --git a/tests/dependency_analyzer/configurations/valid_no_deps.yaml b/tests/dependency_analyzer/configurations/valid_no_deps.yaml index 8ac006c19..d4dd84bc2 100644 --- a/tests/dependency_analyzer/configurations/valid_no_deps.yaml +++ b/tests/dependency_analyzer/configurations/valid_no_deps.yaml @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "id" - path: "https://github.com/owner/name.git" - branch: "master" - digest: "aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7" + id: id + path: https://github.com/owner/name.git + branch: master + digest: aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7 diff --git a/tests/dependency_analyzer/expected_results/osint_maven_FasterXML_jackson-databind.json b/tests/dependency_analyzer/expected_results/cyclonedx_FasterXML_jackson-databind.json similarity index 100% rename from tests/dependency_analyzer/expected_results/osint_maven_FasterXML_jackson-databind.json rename to tests/dependency_analyzer/expected_results/cyclonedx_FasterXML_jackson-databind.json diff --git a/tests/dependency_analyzer/expected_results/osint_maven_micronaut-projects_micronaut-core.json b/tests/dependency_analyzer/expected_results/cyclonedx_micronaut-projects_micronaut-core.json similarity index 100% rename from tests/dependency_analyzer/expected_results/osint_maven_micronaut-projects_micronaut-core.json rename to tests/dependency_analyzer/expected_results/cyclonedx_micronaut-projects_micronaut-core.json diff --git a/tests/dependency_analyzer/expected_results/osint_maven_apache_maven.json b/tests/dependency_analyzer/expected_results/osint_maven_apache_maven.json deleted file mode 100644 index fe1b79df2..000000000 --- a/tests/dependency_analyzer/expected_results/osint_maven_apache_maven.json +++ /dev/null @@ -1 +0,0 @@ -[{"id": "org.junit.jupiter:junit-jupiter-engine", "path": "https://github.com/junit-team/junit5", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.hamcrest:hamcrest-core", "path": "https://github.com/hamcrest/JavaHamcrest", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.eclipse.sisu:org.eclipse.sisu.plexus", "path": "", "branch": "", "digest": "", "note": "Manual configuration required. Could not find SCM URL.", "available": false}, {"id": "org.codehaus.plexus:plexus-utils", "path": "https://github.com/codehaus-plexus/plexus-utils", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.codehaus.plexus:plexus-classworlds", "path": "https://github.com/codehaus-plexus/plexus-classworlds", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.slf4j:slf4j-api", "path": "https://github.com/qos-ch/slf4j", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.apache.maven.shared:maven-shared-utils", "path": "https://github.com/apache/maven-shared-utils", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.apache.maven.resolver:maven-resolver-api", "path": "https://github.com/apache/maven-resolver", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.apache.maven.resolver:maven-resolver-util", "path": "https://github.com/apache/maven-resolver", "branch": "", "digest": "", "note": "https://github.com/apache/maven-resolver is already analyzed.", "available": false}, {"id": "com.google.inject:guice", "path": "https://github.com/google/guice", "branch": "", "digest": "", "note": "", "available": true}, {"id": "com.google.guava:guava", "path": "https://github.com/google/guava", "branch": "", "digest": "", "note": "", "available": true}, {"id": "com.google.guava:failureaccess", "path": "https://github.com/google/guava", "branch": "", "digest": "", "note": "https://github.com/google/guava is already analyzed.", "available": false}, {"id": "javax.inject:javax.inject", "path": "", "branch": "", "digest": "", "note": "Manual configuration required. Could not find SCM URL.", "available": false}, {"id": "javax.annotation:javax.annotation-api", "path": "https://github.com/javaee/javax.annotation", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.codehaus.plexus:plexus-sec-dispatcher", "path": "https://github.com/codehaus-plexus/plexus-sec-dispatcher", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.codehaus.plexus:plexus-cipher", "path": "https://github.com/codehaus-plexus/plexus-cipher", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.slf4j:slf4j-simple", "path": "https://github.com/qos-ch/slf4j", "branch": "", "digest": "", "note": "https://github.com/qos-ch/slf4j is already analyzed.", "available": false}, {"id": "ch.qos.logback:logback-classic", "path": "https://github.com/qos-ch/logback", "branch": "", "digest": "", "note": "", "available": true}, {"id": "commons-cli:commons-cli", "path": "", "branch": "", "digest": "", "note": "Manual configuration required. Could not find SCM URL.", "available": false}, {"id": "org.apache.commons:commons-lang3", "path": "", "branch": "", "digest": "", "note": "Manual configuration required. Could not find SCM URL.", "available": false}, {"id": "org.mockito:mockito-core", "path": "https://github.com/mockito/mockito", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.fusesource.jansi:jansi", "path": "https://github.com/fusesource/jansi", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.apache.maven.wagon:wagon-http", "path": "https://github.com/apache/maven-wagon", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.apache.maven.wagon:wagon-file", "path": "https://github.com/apache/maven-wagon", "branch": "", "digest": "", "note": "https://github.com/apache/maven-wagon is already analyzed.", "available": false}, {"id": "org.slf4j:jcl-over-slf4j", "path": "https://github.com/qos-ch/slf4j", "branch": "", "digest": "", "note": "https://github.com/qos-ch/slf4j is already analyzed.", "available": false}, {"id": "org.apache.maven.resolver:maven-resolver-connector-basic", "path": "https://github.com/apache/maven-resolver", "branch": "", "digest": "", "note": "https://github.com/apache/maven-resolver is already analyzed.", "available": false}, {"id": "org.apache.maven.resolver:maven-resolver-transport-file", "path": "https://github.com/apache/maven-resolver", "branch": "", "digest": "", "note": "https://github.com/apache/maven-resolver is already analyzed.", "available": false}, {"id": "org.apache.maven.resolver:maven-resolver-transport-http", "path": "https://github.com/apache/maven-resolver", "branch": "", "digest": "", "note": "https://github.com/apache/maven-resolver is already analyzed.", "available": false}, {"id": "org.apache.maven.resolver:maven-resolver-transport-wagon", "path": "https://github.com/apache/maven-resolver", "branch": "", "digest": "", "note": "https://github.com/apache/maven-resolver is already analyzed.", "available": false}, {"id": "org.codehaus.plexus:plexus-interpolation", "path": "https://github.com/codehaus-plexus/plexus-interpolation", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.apache.maven.resolver:maven-resolver-impl", "path": "https://github.com/apache/maven-resolver", "branch": "", "digest": "", "note": "https://github.com/apache/maven-resolver is already analyzed.", "available": false}, {"id": "org.codehaus.plexus:plexus-component-annotations", "path": "https://github.com/codehaus-plexus/plexus-containers", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.apache.maven.wagon:wagon-provider-api", "path": "https://github.com/apache/maven-wagon", "branch": "", "digest": "", "note": "https://github.com/apache/maven-wagon is already analyzed.", "available": false}, {"id": "org.codehaus.plexus:plexus-testing", "path": "https://github.com/codehaus-plexus/plexus-testing", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.junit.jupiter:junit-jupiter-params", "path": "https://github.com/junit-team/junit5", "branch": "", "digest": "", "note": "https://github.com/junit-team/junit5 is already analyzed.", "available": false}, {"id": "org.xmlunit:xmlunit-assertj", "path": "https://github.com/xmlunit/xmlunit", "branch": "", "digest": "", "note": "", "available": true}, {"id": "org.hamcrest:hamcrest-library", "path": "https://github.com/hamcrest/JavaHamcrest", "branch": "", "digest": "", "note": "https://github.com/hamcrest/JavaHamcrest is already analyzed.", "available": false}, {"id": "org.eclipse.sisu:org.eclipse.sisu.inject", "path": "", "branch": "", "digest": "", "note": "Manual configuration required. Could not find SCM URL.", "available": false}, {"id": "org.xmlunit:xmlunit-core", "path": "https://github.com/xmlunit/xmlunit", "branch": "", "digest": "", "note": "https://github.com/xmlunit/xmlunit is already analyzed.", "available": false}, {"id": "org.xmlunit:xmlunit-matchers", "path": "https://github.com/xmlunit/xmlunit", "branch": "", "digest": "", "note": "https://github.com/xmlunit/xmlunit is already analyzed.", "available": false}, {"id": "org.apache.maven.resolver:maven-resolver-spi", "path": "https://github.com/apache/maven-resolver", "branch": "", "digest": "", "note": "https://github.com/apache/maven-resolver is already analyzed.", "available": false}, {"id": "commons-jxpath:commons-jxpath", "path": "", "branch": "", "digest": "", "note": "Manual configuration required. Could not find SCM URL.", "available": false}] diff --git a/tests/dependency_analyzer/test_dependency_analyzer.py b/tests/dependency_analyzer/test_dependency_analyzer.py index 89af4d000..dfdcd1ef2 100644 --- a/tests/dependency_analyzer/test_dependency_analyzer.py +++ b/tests/dependency_analyzer/test_dependency_analyzer.py @@ -22,7 +22,6 @@ class TestDependencyAnalyzer(MacaronTestCase): def test_merge_config(self) -> None: """Test merging the manual and automatically resolved configurations.""" - # Mock automatically resolved dependencies. auto_deps = { "com.fasterxml.jackson.core:jackson-annotations": DependencyInfo( diff --git a/tests/e2e/compare_e2e_result.py b/tests/e2e/compare_e2e_result.py index 8a05cf5b5..90530b324 100755 --- a/tests/e2e/compare_e2e_result.py +++ b/tests/e2e/compare_e2e_result.py @@ -31,8 +31,8 @@ def compare_check_results(result: dict, expected: dict) -> int: fail_count += 1 # Compare check results - res_sorted_reqs = sorted(result["results"], key=lambda item: item["check_id"]) - exp_sorted_reqs = sorted(expected["results"], key=lambda item: item["check_id"]) + res_sorted_reqs = sorted(result["results"], key=lambda item: str(item["check_id"])) + exp_sorted_reqs = sorted(expected["results"], key=lambda item: str(item["check_id"])) if len(res_sorted_reqs) < len(exp_sorted_reqs): for req in exp_sorted_reqs[len(res_sorted_reqs) :]: diff --git a/tests/e2e/configurations/jackson_databind_config.yaml b/tests/e2e/configurations/jackson_databind_config.yaml index d88a97eed..8593087d5 100644 --- a/tests/e2e/configurations/jackson_databind_config.yaml +++ b/tests/e2e/configurations/jackson_databind_config.yaml @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "jackson-databind" - branch: "2.14" - digest: "f0af53d085eb2aa9f7f6199846cc526068e09977" - path: "https://github.com/FasterXML/jackson-databind" + id: jackson-databind + branch: '2.14' + digest: f0af53d085eb2aa9f7f6199846cc526068e09977 + path: https://github.com/FasterXML/jackson-databind diff --git a/tests/e2e/configurations/maven_config.yaml b/tests/e2e/configurations/maven_config.yaml index 1202f5e3d..e95a58955 100644 --- a/tests/e2e/configurations/maven_config.yaml +++ b/tests/e2e/configurations/maven_config.yaml @@ -2,18 +2,18 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "apache/maven" - branch: "master" - digest: "6767f2500f1d005924ccff27f04350c253858a84" - path: "https://github.com/apache/maven.git" + id: apache/maven + branch: master + digest: 6767f2500f1d005924ccff27f04350c253858a84 + path: https://github.com/apache/maven.git dependencies: - - id: "guava" - branch: "master" - digest: "d8633ac8539dae52c8361f79c7a0dbd9ad6dd2c4" - path: "https://github.com/google/guava" +- id: guava + branch: master + digest: d8633ac8539dae52c8361f79c7a0dbd9ad6dd2c4 + path: https://github.com/google/guava - - id: "mockito" - branch: "main" - digest: "512ee3949484e4765038a0410cd7a7f1b73cc655" - path: "https://github.com/mockito/mockito" +- id: mockito + branch: main + digest: 512ee3949484e4765038a0410cd7a7f1b73cc655 + path: https://github.com/mockito/mockito diff --git a/tests/e2e/configurations/maven_digest_no_branch.yaml b/tests/e2e/configurations/maven_digest_no_branch.yaml index 670817a9d..f7911a565 100644 --- a/tests/e2e/configurations/maven_digest_no_branch.yaml +++ b/tests/e2e/configurations/maven_digest_no_branch.yaml @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "apache/maven" - branch: "" - digest: "6767f2500f1d005924ccff27f04350c253858a84" - path: "https://github.com/apache/maven.git" + id: apache/maven + branch: '' + digest: 6767f2500f1d005924ccff27f04350c253858a84 + path: https://github.com/apache/maven.git diff --git a/tests/e2e/configurations/maven_invalid_branch.yaml b/tests/e2e/configurations/maven_invalid_branch.yaml index 0b64eb544..192a86251 100644 --- a/tests/e2e/configurations/maven_invalid_branch.yaml +++ b/tests/e2e/configurations/maven_invalid_branch.yaml @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "apache/maven" - branch: "This-branch-does-not-exist" - digest: "" - path: "https://github.com/apache/maven.git" + id: apache/maven + branch: This-branch-does-not-exist + digest: '' + path: https://github.com/apache/maven.git diff --git a/tests/e2e/configurations/maven_invalid_digest.yaml b/tests/e2e/configurations/maven_invalid_digest.yaml index 52ddee02b..be155f612 100644 --- a/tests/e2e/configurations/maven_invalid_digest.yaml +++ b/tests/e2e/configurations/maven_invalid_digest.yaml @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "apache/maven" - branch: "" - digest: "This_digest_is_invalid" - path: "https://github.com/apache/maven.git" + id: apache/maven + branch: '' + digest: This_digest_is_invalid + path: https://github.com/apache/maven.git diff --git a/tests/e2e/configurations/maven_local_path.yaml b/tests/e2e/configurations/maven_local_path.yaml index daad2c881..813dc2841 100644 --- a/tests/e2e/configurations/maven_local_path.yaml +++ b/tests/e2e/configurations/maven_local_path.yaml @@ -2,18 +2,18 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "apache/maven" - branch: "master" - digest: "6767f2500f1d005924ccff27f04350c253858a84" - path: "apache/maven" + id: apache/maven + branch: master + digest: 6767f2500f1d005924ccff27f04350c253858a84 + path: apache/maven dependencies: - - id: "guava" - branch: "master" - digest: "d8633ac8539dae52c8361f79c7a0dbd9ad6dd2c4" - path: "google/guava" +- id: guava + branch: master + digest: d8633ac8539dae52c8361f79c7a0dbd9ad6dd2c4 + path: google/guava - - id: "mockito" - branch: "main" - digest: "512ee3949484e4765038a0410cd7a7f1b73cc655" - path: "mockito/mockito" +- id: mockito + branch: main + digest: 512ee3949484e4765038a0410cd7a7f1b73cc655 + path: mockito/mockito diff --git a/tests/e2e/configurations/micronaut_core_config.yaml b/tests/e2e/configurations/micronaut_core_config.yaml index 2f64186b8..1eb8bb557 100644 --- a/tests/e2e/configurations/micronaut_core_config.yaml +++ b/tests/e2e/configurations/micronaut_core_config.yaml @@ -2,24 +2,24 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: - id: "micronaut-core" + id: micronaut-core # For version 3.5.5 # https://github.com/micronaut-projects/micronaut-core/commit/08357218ce6d6347c4d1e3ff74a8b2936fe3c3dc - branch: "3.5.x" - digest: "08357218ce6d6347c4d1e3ff74a8b2936fe3c3dc" - path: "https://github.com/micronaut-projects/micronaut-core" + branch: 3.5.x + digest: 08357218ce6d6347c4d1e3ff74a8b2936fe3c3dc + path: https://github.com/micronaut-projects/micronaut-core dependencies: - - id: "slf4j" +- id: slf4j # For version 1.7.36 # https://github.com/qos-ch/slf4j/commit/e9ee55cca93c2bf26f14482a9bdf961c750d2a56 - branch: "v_1.7.36" - digest: "e9ee55cca93c2bf26f14482a9bdf961c750d2a56" - path: "https://github.com/qos-ch/slf4j.git" + branch: v_1.7.36 + digest: e9ee55cca93c2bf26f14482a9bdf961c750d2a56 + path: https://github.com/qos-ch/slf4j.git - - id: "caffeine" +- id: caffeine # For version 2.9.3 # https://github.com/ben-manes/caffeine/commit/05a040c2478341bab8a58a02b3dc1fe14d626d72 - branch: "v2.9.3" - digest: "05a040c2478341bab8a58a02b3dc1fe14d626d72" - path: "https://github.com/ben-manes/caffeine.git" + branch: v2.9.3 + digest: 05a040c2478341bab8a58a02b3dc1fe14d626d72 + path: https://github.com/ben-manes/caffeine.git diff --git a/tests/macaron_testcase.py b/tests/macaron_testcase.py index faede946b..48caf78cd 100644 --- a/tests/macaron_testcase.py +++ b/tests/macaron_testcase.py @@ -21,7 +21,7 @@ class MacaronTestCase(TestCase): @classmethod def setUpClass(cls) -> None: - """Setup the necessary values for the tests.""" + """Set up the necessary values for the tests.""" # Load values from defaults.ini. if not cls.macaron_test_dir.joinpath("defaults.ini").exists(): create_defaults(str(cls.macaron_test_dir), str(cls.macaron_path)) diff --git a/tests/parsers/actionparser/resources/expected_results/codeql-analysis.json b/tests/parsers/actionparser/resources/expected_results/codeql-analysis.json index d2086a0ca..324d851fa 100644 --- a/tests/parsers/actionparser/resources/expected_results/codeql-analysis.json +++ b/tests/parsers/actionparser/resources/expected_results/codeql-analysis.json @@ -1,7 +1,7 @@ { "Name": { "Value": "CodeQL", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 4, "Col": 7 @@ -13,10 +13,10 @@ "Cron": [ { "Value": "40 21 * * 2", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 8, - "Col": 13 + "Col": 11 } } ], @@ -95,7 +95,7 @@ "Values": [ { "Value": "README.md", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 16, "Col": 7 @@ -103,7 +103,7 @@ }, { "Value": "release-notes/*", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 17, "Col": 7 @@ -187,7 +187,7 @@ "Values": [ { "Value": "README.md", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 25, "Col": 7 @@ -195,7 +195,7 @@ }, { "Value": "release-notes/*", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 26, "Col": 7 diff --git a/tests/parsers/actionparser/resources/expected_results/maven.json b/tests/parsers/actionparser/resources/expected_results/maven.json index f591c6c2e..8d522acb6 100644 --- a/tests/parsers/actionparser/resources/expected_results/maven.json +++ b/tests/parsers/actionparser/resources/expected_results/maven.json @@ -99,7 +99,7 @@ "Quoted": false, "Pos": { "Line": 18, - "Col": 15 + "Col": 13 } }, "Inputs": null, @@ -111,7 +111,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 18, - "Col": 9 + "Col": 7 } }, { @@ -124,7 +124,7 @@ "Quoted": false, "Pos": { "Line": 19, - "Col": 15 + "Col": 13 } }, "Inputs": { @@ -134,15 +134,15 @@ "Quoted": false, "Pos": { "Line": 23, - "Col": 11 + "Col": 9 } }, "Value": { "Value": "maven", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 23, - "Col": 18 + "Col": 16 } } }, @@ -152,15 +152,15 @@ "Quoted": false, "Pos": { "Line": 22, - "Col": 11 + "Col": 9 } }, "Value": { "Value": "temurin", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 22, - "Col": 25 + "Col": 23 } } }, @@ -170,7 +170,7 @@ "Quoted": false, "Pos": { "Line": 21, - "Col": 11 + "Col": 9 } }, "Value": { @@ -178,7 +178,7 @@ "Quoted": false, "Pos": { "Line": 21, - "Col": 25 + "Col": 23 } } } @@ -191,7 +191,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 19, - "Col": 9 + "Col": 7 } }, { @@ -202,7 +202,7 @@ "Quoted": false, "Pos": { "Line": 25, - "Col": 15 + "Col": 13 } }, "Exec": { @@ -211,14 +211,14 @@ "Quoted": false, "Pos": { "Line": 26, - "Col": 14 + "Col": 12 } }, "Shell": null, "WorkingDirectory": null, "RunPos": { "Line": 26, - "Col": 9 + "Col": 7 } }, "Env": null, @@ -226,7 +226,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 25, - "Col": 9 + "Col": 7 } }, { @@ -236,7 +236,7 @@ "Quoted": false, "Pos": { "Line": 30, - "Col": 13 + "Col": 11 } }, "Name": { @@ -244,7 +244,7 @@ "Quoted": false, "Pos": { "Line": 28, - "Col": 15 + "Col": 13 } }, "Exec": { @@ -253,7 +253,7 @@ "Quoted": false, "Pos": { "Line": 29, - "Col": 15 + "Col": 13 } }, "Inputs": { @@ -263,7 +263,7 @@ "Quoted": false, "Pos": { "Line": 32, - "Col": 11 + "Col": 9 } }, "Value": { @@ -271,7 +271,7 @@ "Quoted": false, "Pos": { "Line": 32, - "Col": 17 + "Col": 15 } } }, @@ -281,7 +281,7 @@ "Quoted": false, "Pos": { "Line": 33, - "Col": 11 + "Col": 9 } }, "Value": { @@ -289,7 +289,7 @@ "Quoted": false, "Pos": { "Line": 33, - "Col": 17 + "Col": 15 } } } @@ -302,7 +302,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 28, - "Col": 9 + "Col": 7 } } ], @@ -411,7 +411,7 @@ "Quoted": false, "Pos": { "Line": 46, - "Col": 15 + "Col": 13 } }, "Exec": { @@ -420,7 +420,7 @@ "Quoted": false, "Pos": { "Line": 50, - "Col": 14 + "Col": 12 } }, "Shell": { @@ -428,13 +428,13 @@ "Quoted": false, "Pos": { "Line": 47, - "Col": 16 + "Col": 14 } }, "WorkingDirectory": null, "RunPos": { "Line": 50, - "Col": 9 + "Col": 7 } }, "Env": { @@ -445,7 +445,7 @@ "Quoted": false, "Pos": { "Line": 49, - "Col": 11 + "Col": 9 } }, "Value": { @@ -453,7 +453,7 @@ "Quoted": false, "Pos": { "Line": 49, - "Col": 26 + "Col": 24 } } } @@ -464,7 +464,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 46, - "Col": 9 + "Col": 7 } }, { @@ -475,7 +475,7 @@ "Quoted": false, "Pos": { "Line": 77, - "Col": 15 + "Col": 13 } }, "Exec": { @@ -484,7 +484,7 @@ "Quoted": false, "Pos": { "Line": 78, - "Col": 15 + "Col": 13 } }, "Inputs": { @@ -494,7 +494,7 @@ "Quoted": false, "Pos": { "Line": 81, - "Col": 11 + "Col": 9 } }, "Value": { @@ -502,7 +502,7 @@ "Quoted": false, "Pos": { "Line": 81, - "Col": 17 + "Col": 15 } } }, @@ -512,7 +512,7 @@ "Quoted": false, "Pos": { "Line": 82, - "Col": 11 + "Col": 9 } }, "Value": { @@ -520,7 +520,7 @@ "Quoted": false, "Pos": { "Line": 82, - "Col": 16 + "Col": 14 } } }, @@ -530,7 +530,7 @@ "Quoted": false, "Pos": { "Line": 80, - "Col": 11 + "Col": 9 } }, "Value": { @@ -538,7 +538,7 @@ "Quoted": false, "Pos": { "Line": 80, - "Col": 23 + "Col": 21 } } } @@ -551,7 +551,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 77, - "Col": 9 + "Col": 7 } }, { @@ -562,7 +562,7 @@ "Quoted": false, "Pos": { "Line": 84, - "Col": 15 + "Col": 13 } }, "Exec": { @@ -571,7 +571,7 @@ "Quoted": false, "Pos": { "Line": 85, - "Col": 15 + "Col": 13 } }, "Inputs": { @@ -581,7 +581,7 @@ "Quoted": false, "Pos": { "Line": 88, - "Col": 11 + "Col": 9 } }, "Value": { @@ -589,7 +589,7 @@ "Quoted": false, "Pos": { "Line": 88, - "Col": 16 + "Col": 14 } } }, @@ -599,7 +599,7 @@ "Quoted": false, "Pos": { "Line": 87, - "Col": 11 + "Col": 9 } }, "Value": { @@ -607,7 +607,7 @@ "Quoted": false, "Pos": { "Line": 87, - "Col": 17 + "Col": 15 } } }, @@ -617,7 +617,7 @@ "Quoted": false, "Pos": { "Line": 89, - "Col": 11 + "Col": 9 } }, "Value": { @@ -625,7 +625,7 @@ "Quoted": false, "Pos": { "Line": 89, - "Col": 25 + "Col": 23 } } } @@ -638,7 +638,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 84, - "Col": 9 + "Col": 7 } }, { @@ -649,7 +649,7 @@ "Quoted": false, "Pos": { "Line": 92, - "Col": 15 + "Col": 13 } }, "Exec": { @@ -658,7 +658,7 @@ "Quoted": false, "Pos": { "Line": 93, - "Col": 15 + "Col": 13 } }, "Inputs": { @@ -668,7 +668,7 @@ "Quoted": false, "Pos": { "Line": 95, - "Col": 11 + "Col": 9 } }, "Value": { @@ -676,7 +676,7 @@ "Quoted": false, "Pos": { "Line": 95, - "Col": 17 + "Col": 15 } } }, @@ -686,7 +686,7 @@ "Quoted": false, "Pos": { "Line": 96, - "Col": 11 + "Col": 9 } }, "Value": { @@ -694,7 +694,7 @@ "Quoted": false, "Pos": { "Line": 96, - "Col": 17 + "Col": 15 } } } @@ -707,7 +707,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 92, - "Col": 9 + "Col": 7 } }, { @@ -718,7 +718,7 @@ "Quoted": false, "Pos": { "Line": 98, - "Col": 15 + "Col": 13 } }, "Exec": { @@ -727,7 +727,7 @@ "Quoted": false, "Pos": { "Line": 99, - "Col": 15 + "Col": 13 } }, "Inputs": { @@ -737,15 +737,15 @@ "Quoted": false, "Pos": { "Line": 103, - "Col": 11 + "Col": 9 } }, "Value": { "Value": "maven", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 103, - "Col": 18 + "Col": 16 } } }, @@ -755,15 +755,15 @@ "Quoted": false, "Pos": { "Line": 102, - "Col": 11 + "Col": 9 } }, "Value": { "Value": "temurin", - "Quoted": true, + "Quoted": false, "Pos": { "Line": 102, - "Col": 25 + "Col": 23 } } }, @@ -773,7 +773,7 @@ "Quoted": false, "Pos": { "Line": 101, - "Col": 11 + "Col": 9 } }, "Value": { @@ -781,7 +781,7 @@ "Quoted": false, "Pos": { "Line": 101, - "Col": 25 + "Col": 23 } } } @@ -794,7 +794,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 98, - "Col": 9 + "Col": 7 } }, { @@ -805,7 +805,7 @@ "Quoted": false, "Pos": { "Line": 105, - "Col": 15 + "Col": 13 } }, "Exec": { @@ -814,7 +814,7 @@ "Quoted": false, "Pos": { "Line": 107, - "Col": 14 + "Col": 12 } }, "Shell": { @@ -822,13 +822,13 @@ "Quoted": false, "Pos": { "Line": 106, - "Col": 16 + "Col": 14 } }, "WorkingDirectory": null, "RunPos": { "Line": 107, - "Col": 9 + "Col": 7 } }, "Env": null, @@ -836,7 +836,7 @@ "TimeoutMinutes": null, "Pos": { "Line": 105, - "Col": 9 + "Col": 7 } } ], diff --git a/tests/parsers/actionparser/resources/expected_results/release.json b/tests/parsers/actionparser/resources/expected_results/release.json index dda531bca..f26d8e33b 100644 --- a/tests/parsers/actionparser/resources/expected_results/release.json +++ b/tests/parsers/actionparser/resources/expected_results/release.json @@ -90,7 +90,7 @@ } }, "Value": { - "Value": "artifact-ubuntu-latest-3.10", + "Value": "artifact-ubuntu-latest-3.11", "Quoted": false, "Pos": { "Line": 12, diff --git a/tests/parsers/actionparser/resources/workflow_files/codeql-analysis.yml b/tests/parsers/actionparser/resources/workflow_files/codeql-analysis.yml index 7c9523e75..9d5e6ea57 100644 --- a/tests/parsers/actionparser/resources/workflow_files/codeql-analysis.yml +++ b/tests/parsers/actionparser/resources/workflow_files/codeql-analysis.yml @@ -1,29 +1,29 @@ # Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -name: "CodeQL" +name: CodeQL on: schedule: - - cron: '40 21 * * 2' + - cron: 40 21 * * 2 push: branches: - master - - "3.0" - - "2.14" - - "2.13" + - '3.0' + - '2.14' + - '2.13' paths-ignore: - - "README.md" - - "release-notes/*" + - README.md + - release-notes/* pull_request: branches: - master - - "3.0" - - "2.14" - - "2.13" + - '3.0' + - '2.14' + - '2.13' paths-ignore: - - "README.md" - - "release-notes/*" + - README.md + - release-notes/* jobs: analyze: @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'java' ] + language: [java] steps: - name: Checkout repository diff --git a/tests/parsers/actionparser/resources/workflow_files/maven.yml b/tests/parsers/actionparser/resources/workflow_files/maven.yml index 92fe2005b..997096c4f 100644 --- a/tests/parsers/actionparser/resources/workflow_files/maven.yml +++ b/tests/parsers/actionparser/resources/workflow_files/maven.yml @@ -15,22 +15,22 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v2 - with: - java-version: 8 - distribution: 'temurin' - cache: 'maven' + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + java-version: 8 + distribution: temurin + cache: maven - - name: Build with Maven - run: mvn verify -e -B -V -DdistributionFileName=apache-maven + - name: Build with Maven + run: mvn verify -e -B -V -DdistributionFileName=apache-maven - - name: Upload built Maven - uses: actions/upload-artifact@v2 - if: ${{ matrix.os == 'ubuntu-latest' }} - with: - name: built-maven - path: apache-maven/target/ + - name: Upload built Maven + uses: actions/upload-artifact@v2 + if: ${{ matrix.os == 'ubuntu-latest' }} + with: + name: built-maven + path: apache-maven/target/ integration-test: needs: build @@ -43,65 +43,65 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Collect environment context variables - shell: bash - env: - PR_HEAD_LABEL: ${{ github.event.pull_request.head.label }} - run: | - set +e - repo=maven-integration-testing - target_branch=master - target_user=apache - if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then - user=${PR_HEAD_LABEL%:*} - branch=${PR_HEAD_LABEL#*:} + - name: Collect environment context variables + shell: bash + env: + PR_HEAD_LABEL: ${{ github.event.pull_request.head.label }} + run: | + set +e + repo=maven-integration-testing + target_branch=master + target_user=apache + if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + user=${PR_HEAD_LABEL%:*} + branch=${PR_HEAD_LABEL#*:} + else + user=${GITHUB_REPOSITORY%/*} + branch=${GITHUB_REF#refs/heads/} + fi + if [ $branch != "master" ]; then + git ls-remote https://github.com/$user/$repo.git | grep "refs/heads/${branch}$" > /dev/null + if [ $? -eq 0 ]; then + echo "Found a branch \"$branch\" in fork \"$user/$repo\", configuring this for the integration tests to be run against." + target_branch=$branch + target_user=$user else - user=${GITHUB_REPOSITORY%/*} - branch=${GITHUB_REF#refs/heads/} + echo "Could not find fork \"$user/$repo\" or a branch \"$branch\" in this fork. Falling back to \"$target_branch\" in \"$target_user/$repo\"." fi - if [ $branch != "master" ]; then - git ls-remote https://github.com/$user/$repo.git | grep "refs/heads/${branch}$" > /dev/null - if [ $? -eq 0 ]; then - echo "Found a branch \"$branch\" in fork \"$user/$repo\", configuring this for the integration tests to be run against." - target_branch=$branch - target_user=$user - else - echo "Could not find fork \"$user/$repo\" or a branch \"$branch\" in this fork. Falling back to \"$target_branch\" in \"$target_user/$repo\"." - fi - else - echo "Integration tests will run against $target_user/$repo for master builds." - fi - echo "REPO_BRANCH=$target_branch" >> $GITHUB_ENV - echo "REPO_USER=$target_user" >> $GITHUB_ENV + else + echo "Integration tests will run against $target_user/$repo for master builds." + fi + echo "REPO_BRANCH=$target_branch" >> $GITHUB_ENV + echo "REPO_USER=$target_user" >> $GITHUB_ENV - - name: Checkout maven-integration-testing - uses: actions/checkout@v2 - with: - repository: ${{ env.REPO_USER }}/maven-integration-testing - path: maven-integration-testing/ - ref: ${{ env.REPO_BRANCH }} + - name: Checkout maven-integration-testing + uses: actions/checkout@v2 + with: + repository: ${{ env.REPO_USER }}/maven-integration-testing + path: maven-integration-testing/ + ref: ${{ env.REPO_BRANCH }} - - name: Set up cache for ~/.m2/repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: it-m2-repo-${{ matrix.os }}-${{ hashFiles('maven-integration-testing/**/pom.xml') }} - restore-keys: | - it-m2-repo-${{ matrix.os }}- + - name: Set up cache for ~/.m2/repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: it-m2-repo-${{ matrix.os }}-${{ hashFiles('maven-integration-testing/**/pom.xml') }} + restore-keys: | + it-m2-repo-${{ matrix.os }}- - - name: Download built Maven - uses: actions/download-artifact@v2 - with: - name: built-maven - path: built-maven/ + - name: Download built Maven + uses: actions/download-artifact@v2 + with: + name: built-maven + path: built-maven/ - - name: Set up JDK - uses: actions/setup-java@v2 - with: - java-version: ${{ matrix.java }} - distribution: 'temurin' - cache: 'maven' + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.java }} + distribution: temurin + cache: maven - - name: Running integration tests - shell: bash - run: mvn install -e -B -V -Prun-its,embedded -Dmaven.repo.local="$HOME/.m2/repository" -DmavenDistro="$GITHUB_WORKSPACE/built-maven/apache-maven-bin.zip" -f maven-integration-testing/pom.xml + - name: Running integration tests + shell: bash + run: mvn install -e -B -V -Prun-its,embedded -Dmaven.repo.local="$HOME/.m2/repository" -DmavenDistro="$GITHUB_WORKSPACE/built-maven/apache-maven-bin.zip" -f maven-integration-testing/pom.xml diff --git a/tests/parsers/actionparser/resources/workflow_files/release.yaml b/tests/parsers/actionparser/resources/workflow_files/release.yaml index 126b19da7..1f06f7755 100644 --- a/tests/parsers/actionparser/resources/workflow_files/release.yaml +++ b/tests/parsers/actionparser/resources/workflow_files/release.yaml @@ -9,7 +9,7 @@ on: permissions: contents: read env: - ARTIFACT_NAME: artifact-ubuntu-latest-3.10 + ARTIFACT_NAME: artifact-ubuntu-latest-3.11 jobs: build: diff --git a/tests/parsers/actionparser/test_actionparser.py b/tests/parsers/actionparser/test_actionparser.py index 1574e9ae2..21819bfe8 100644 --- a/tests/parsers/actionparser/test_actionparser.py +++ b/tests/parsers/actionparser/test_actionparser.py @@ -19,7 +19,6 @@ class TestParsers(MacaronTestCase): def test_actionparser_parse(self) -> None: """Test parsing GH Actions workflows.""" - resources_dir = Path(__file__).parent.joinpath("resources") valid_results = [] diff --git a/tests/parsers/bashparser/test_bashparser.py b/tests/parsers/bashparser/test_bashparser.py index 2b258612a..6b1a4e3f9 100644 --- a/tests/parsers/bashparser/test_bashparser.py +++ b/tests/parsers/bashparser/test_bashparser.py @@ -19,7 +19,6 @@ class TestParsers(MacaronTestCase): def test_bashparser_parse(self) -> None: """Test parsing bash scripts.""" - resources_dir = Path(__file__).parent.joinpath("resources") # Parse the valid mock bash script. @@ -41,4 +40,4 @@ def test_bashparser_parse(self) -> None: # Parse invalid workflows. with open(os.path.join(resources_dir, "bash_files", "invalid.sh"), encoding="utf8") as bash_file: - assert parse(bash_file.read(), str(MacaronTestCase.macaron_path)) == {} + assert not parse(bash_file.read(), str(MacaronTestCase.macaron_path)) diff --git a/tests/parsers/yaml/resources/invalid.yaml b/tests/parsers/yaml/resources/invalid.yaml index 0098a7a0f..397cd0fee 100644 --- a/tests/parsers/yaml/resources/invalid.yaml +++ b/tests/parsers/yaml/resources/invalid.yaml @@ -2,4 +2,4 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. # Not a valid yaml file. -: +null: diff --git a/tests/parsers/yaml/resources/not_valid_against_schema.yaml b/tests/parsers/yaml/resources/not_valid_against_schema.yaml index 5905840cf..803230159 100644 --- a/tests/parsers/yaml/resources/not_valid_against_schema.yaml +++ b/tests/parsers/yaml/resources/not_valid_against_schema.yaml @@ -4,17 +4,17 @@ # This is a valid yaml and match the schema. target: - id: "id" - path: "https://github.com/owner/name.git" - branch: "master" - digest: "aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7" + id: id + path: https://github.com/owner/name.git + branch: master + digest: aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7 dependencies: - - id: "id" - path: "https://github.com/owner/name.git" - branch: "master" - digest: "aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7" - - id: "id" - path: "https://github.com/owner/name.git" - branch: "master" - digest: "aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7" +- id: id + path: https://github.com/owner/name.git + branch: master + digest: aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7 +- id: id + path: https://github.com/owner/name.git + branch: master + digest: aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7 diff --git a/tests/parsers/yaml/resources/schema.yaml b/tests/parsers/yaml/resources/schema.yaml index 96c0ada7d..dd9463cfc 100644 --- a/tests/parsers/yaml/resources/schema.yaml +++ b/tests/parsers/yaml/resources/schema.yaml @@ -1,3 +1,4 @@ +--- # Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/parsers/yaml/resources/valid_against_schema.yaml b/tests/parsers/yaml/resources/valid_against_schema.yaml index 0a86534f0..d19b0410c 100644 --- a/tests/parsers/yaml/resources/valid_against_schema.yaml +++ b/tests/parsers/yaml/resources/valid_against_schema.yaml @@ -5,5 +5,5 @@ target: # Missing id and path. - branch: "master" - digest: "aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7" + branch: master + digest: aac3b3bcb608e1e8451d4beedd38ecbe6306e7e7 diff --git a/tests/parsers/yaml/test_yaml_loader.py b/tests/parsers/yaml/test_yaml_loader.py index 24c09f837..8e6b5f74c 100644 --- a/tests/parsers/yaml/test_yaml_loader.py +++ b/tests/parsers/yaml/test_yaml_loader.py @@ -30,11 +30,11 @@ def test_load_yaml_content(self) -> None: # Failed while loading the yaml file with patch("yamale.make_data", side_effect=YAMLError): - assert YamlLoader._load_yaml_content("sample_file_path") == [] + assert not YamlLoader._load_yaml_content("sample_file_path") # File not found with patch("yamale.make_data", side_effect=FileNotFoundError): - assert YamlLoader._load_yaml_content("sample_file_path") == [] + assert not YamlLoader._load_yaml_content("sample_file_path") def test_validate_yaml_data(self) -> None: """Test the validate yaml data method.""" @@ -56,7 +56,7 @@ def test_load(self) -> None: schema_file = os.path.join(self.RESOURCES_DIR, "schema.yaml") schema: Schema = yamale.make_schema(schema_file) - assert not YamlLoader.load(os.path.join(self.RESOURCES_DIR, "invalid.yaml")) + assert YamlLoader.load(os.path.join(self.RESOURCES_DIR, "invalid.yaml")) == {None: None} assert not YamlLoader.load(os.path.join(self.RESOURCES_DIR, "invalid.yaml"), schema) assert YamlLoader.load(os.path.join(self.RESOURCES_DIR, "valid_against_schema.yaml")) diff --git a/tests/policy_engine/resources/policies/invalid.yaml b/tests/policy_engine/resources/policies/invalid.yaml index 81ee3e28d..62a3bf5f4 100644 --- a/tests/policy_engine/resources/policies/invalid.yaml +++ b/tests/policy_engine/resources/policies/invalid.yaml @@ -3,7 +3,7 @@ metadata: id: MACARON_1 - description: "Missing definition policy." + description: Missing definition policy. # Missing the definition # definition: diff --git a/tests/policy_engine/resources/policies/slsa_verifier.yaml b/tests/policy_engine/resources/policies/slsa_verifier.yaml index 1f1424b57..c6096d535 100644 --- a/tests/policy_engine/resources/policies/slsa_verifier.yaml +++ b/tests/policy_engine/resources/policies/slsa_verifier.yaml @@ -3,7 +3,7 @@ metadata: id: MACARON_1 - description: "Slsa-verifier policy - SLSA provenance v0.2." + description: Slsa-verifier policy - SLSA provenance v0.2. definition: _type: https://in-toto.io/Statement/v0.1 diff --git a/tests/policy_engine/test_policy.py b/tests/policy_engine/test_policy.py index 07ea179c2..568c72db0 100644 --- a/tests/policy_engine/test_policy.py +++ b/tests/policy_engine/test_policy.py @@ -31,7 +31,6 @@ def test_get_policy_from_file(self) -> None: # pylint: disable=not-callable def test_validating_data(self) -> None: """Test validating data using the function returned by gen_policy_func.""" - # float('nan') is not equal to itself. assert not _gen_policy_func({"A": float("nan")})({"A": float("nan")}) diff --git a/tests/slsa_analyzer/checks/base_check/test_base_check.py b/tests/slsa_analyzer/checks/base_check/test_base_check.py index 1763778d3..d3600babc 100644 --- a/tests/slsa_analyzer/checks/base_check/test_base_check.py +++ b/tests/slsa_analyzer/checks/base_check/test_base_check.py @@ -14,7 +14,6 @@ class TestConfiguration(TestCase): def test_raise_implementation_error(self) -> None: """Test raising errors if child class does not override abstract method(s).""" - # pylint: disable=abstract-method class ChildCheck(BaseCheck): """This class is a child class that does not implement abstract methods in Base Check.""" diff --git a/tests/slsa_analyzer/checks/resources/github/workflow_files/maven_build_itself.yml b/tests/slsa_analyzer/checks/resources/github/workflow_files/maven_build_itself.yml index ce779301f..40b8f6f6c 100644 --- a/tests/slsa_analyzer/checks/resources/github/workflow_files/maven_build_itself.yml +++ b/tests/slsa_analyzer/checks/resources/github/workflow_files/maven_build_itself.yml @@ -16,44 +16,44 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v2 - with: - java-version: ${{ matrix.java }} - distribution: 'temurin' - cache: 'maven' - - - name: Build with Maven - run: mvn verify -e -B -V -DdistributionFileName=apache-maven - - - name: Extract tarball - shell: bash - run: | - set +e - if [ -f ${{ env.TAR_BALL }} ]; then - temp_dir=$(mktemp -d) - tar -xzf ${{ env.TAR_BALL }} -C "$temp_dir" --strip 1 - maven_bin_dir=$temp_dir/bin - if [ -d $maven_bin_dir ]; then - echo "tar.gz file \"${{ env.TAR_BALL }}\" succesfully extracted in temporarily directory \"$temp_dir.\"" - echo "TEMP_MAVEN_BIN_DIR=$maven_bin_dir" >> $GITHUB_ENV - else - echo "$maven_bin_dir does not exist." - exit 1; - fi + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.java }} + distribution: temurin + cache: maven + + - name: Build with Maven + run: mvn verify -e -B -V -DdistributionFileName=apache-maven + + - name: Extract tarball + shell: bash + run: | + set +e + if [ -f ${{ env.TAR_BALL }} ]; then + temp_dir=$(mktemp -d) + tar -xzf ${{ env.TAR_BALL }} -C "$temp_dir" --strip 1 + maven_bin_dir=$temp_dir/bin + if [ -d $maven_bin_dir ]; then + echo "tar.gz file \"${{ env.TAR_BALL }}\" succesfully extracted in temporarily directory \"$temp_dir.\"" + echo "TEMP_MAVEN_BIN_DIR=$maven_bin_dir" >> $GITHUB_ENV else - echo "${{ env.TAR_BALL }} does not exist." + echo "$maven_bin_dir does not exist." exit 1; fi - env: - TAR_BALL: apache-maven/target/apache-maven-bin.tar.gz - - - name: Clean with Maven - run: mvn clean - - - name: Build again with Maven SNAPSHOT - shell: bash - run: | - set +e - export PATH=${{ env.TEMP_MAVEN_BIN_DIR }}:$PATH - mvn verify -e -B -V -DdistributionFileName=apache-maven + else + echo "${{ env.TAR_BALL }} does not exist." + exit 1; + fi + env: + TAR_BALL: apache-maven/target/apache-maven-bin.tar.gz + + - name: Clean with Maven + run: mvn clean + + - name: Build again with Maven SNAPSHOT + shell: bash + run: | + set +e + export PATH=${{ env.TEMP_MAVEN_BIN_DIR }}:$PATH + mvn verify -e -B -V -DdistributionFileName=apache-maven diff --git a/tests/slsa_analyzer/checks/resources/github/workflow_files/slsa_verifier.yaml b/tests/slsa_analyzer/checks/resources/github/workflow_files/slsa_verifier.yaml index 8fc5b3d1b..caab826e1 100644 --- a/tests/slsa_analyzer/checks/resources/github/workflow_files/slsa_verifier.yaml +++ b/tests/slsa_analyzer/checks/resources/github/workflow_files/slsa_verifier.yaml @@ -8,7 +8,7 @@ on: workflow_dispatch: push: tags: - - "*" # triggers only if push new tag version, like `0.8.4`. + - '*' # triggers only if push new tag version, like `0.8.4`. permissions: read-all diff --git a/tests/slsa_analyzer/checks/test_gradle_build_tool.py b/tests/slsa_analyzer/checks/test_gradle_build_tool.py index 3731d8363..26b2b3faf 100644 --- a/tests/slsa_analyzer/checks/test_gradle_build_tool.py +++ b/tests/slsa_analyzer/checks/test_gradle_build_tool.py @@ -7,8 +7,9 @@ import os from pathlib import Path +import pytest + from macaron.slsa_analyzer.build_tool import Gradle -from macaron.slsa_analyzer.build_tool.base_build_tool import _find_parent_file_in from ...macaron_testcase import MacaronTestCase from ..mock_git_utils import prepare_repo_for_testing @@ -17,6 +18,7 @@ class TestGradleBuildTool(MacaronTestCase): """Test the gradle build tool.""" + @pytest.mark.skip() def test_gradle_build_tool(self) -> None: """Test the gradle build tool.""" base_dir = Path(__file__).parent @@ -38,15 +40,9 @@ def test_gradle_build_tool(self) -> None: # A repo without gradle assert not gradle_tool.is_detected(no_gradle.git_obj.path) - assert not _find_parent_file_in(no_gradle.git_obj.path, "settings.gradle") - assert not _find_parent_file_in(no_gradle.git_obj.path, "settings.gradle.kts") # A repo with groovy gradle assert gradle_tool.is_detected(groovy_gradle.git_obj.path) - assert _find_parent_file_in(groovy_gradle.git_obj.path, "settings.gradle") - assert not _find_parent_file_in(groovy_gradle.git_obj.path, "settings.gradle.kts") # A repo with kotlin gradle assert gradle_tool.is_detected(kotlin_gradle.git_obj.path) - assert not _find_parent_file_in(kotlin_gradle.git_obj.path, "settings.gradle") - assert _find_parent_file_in(kotlin_gradle.git_obj.path, "settings.gradle.kts") diff --git a/tests/slsa_analyzer/checks/test_maven_build_tool.py b/tests/slsa_analyzer/checks/test_maven_build_tool.py index e5181a513..be1f057bb 100644 --- a/tests/slsa_analyzer/checks/test_maven_build_tool.py +++ b/tests/slsa_analyzer/checks/test_maven_build_tool.py @@ -6,8 +6,9 @@ import os from pathlib import Path +import pytest + from macaron.slsa_analyzer.build_tool import Maven -from macaron.slsa_analyzer.build_tool.base_build_tool import _find_parent_file_in from ...macaron_testcase import MacaronTestCase from ..mock_git_utils import prepare_repo_for_testing @@ -16,6 +17,7 @@ class TestMavenBuildTool(MacaronTestCase): """Test the Maven build tool.""" + @pytest.mark.skip() def test_maven_build_tool(self) -> None: """Test the Maven build tool.""" base_dir = Path(__file__).parent @@ -35,14 +37,11 @@ def test_maven_build_tool(self) -> None: # A repo with no pom no_pom_repo = prepare_repo_for_testing(no_pom, self.macaron_path, base_dir) assert not maven_tool.is_detected(no_pom_repo.git_obj.path) - assert not _find_parent_file_in(no_pom_repo.git_obj.path, "pom.xml") # A repo with pom for each sub-module but no parent pom no_parent_pom_repo = prepare_repo_for_testing(no_parent_pom, self.macaron_path, base_dir) assert maven_tool.is_detected(no_parent_pom_repo.git_obj.path) - assert not _find_parent_file_in(no_parent_pom_repo.git_obj.path, "pom.xml") # A repo with pom for each sub-module and parent pom has_parent_pom_repo = prepare_repo_for_testing(has_parent_pom, self.macaron_path, base_dir) assert maven_tool.is_detected(has_parent_pom_repo.git_obj.path) - assert _find_parent_file_in(has_parent_pom_repo.git_obj.path, "pom.xml") diff --git a/tests/slsa_analyzer/checks/test_policy_check.py b/tests/slsa_analyzer/checks/test_policy_check.py index 82f46bd31..06b87f123 100644 --- a/tests/slsa_analyzer/checks/test_policy_check.py +++ b/tests/slsa_analyzer/checks/test_policy_check.py @@ -59,7 +59,6 @@ class TestPolicyCheck(MacaronTestCase): def test_policy_check(self) -> None: """Test the policy check.""" - check = PolicyCheck() check_result = CheckResult(justification=[]) # type: ignore github_actions = MockGitHubActions() diff --git a/tests/slsa_analyzer/checks/test_registry.py b/tests/slsa_analyzer/checks/test_registry.py index 5ff955cc4..179d12c8d 100644 --- a/tests/slsa_analyzer/checks/test_registry.py +++ b/tests/slsa_analyzer/checks/test_registry.py @@ -31,14 +31,12 @@ def setUp(self) -> None: def test_exit_on_duplicated(self) -> None: """Test registering a duplicated check_id Check.""" - with self.assertRaises(SystemExit): self.REGISTRY.register(BaseCheck("mcn_duplicated_check_1", "")) # type: ignore self.REGISTRY.register(BaseCheck("mcn_duplicated_check_1", "")) # type: ignore def test_exit_on_empty_check_id(self) -> None: """Test registering an empty check_id Check.""" - with self.assertRaises(SystemExit): self.REGISTRY.register(BaseCheck("", "")) # type: ignore @@ -50,7 +48,6 @@ def test_exit_on_invalid_registered_check(self, check: SearchStrategy) -> None: def test_add_successfully(self) -> None: """Test registering a Check correctly.""" - self.REGISTRY.register(BaseCheck("mcn_correct_check_1", "This check is a correct Check.")) # type: ignore assert self.REGISTRY.get_all_checks_mapping().get("mcn_correct_check_1") diff --git a/tests/slsa_analyzer/checks/test_vcs_check.py b/tests/slsa_analyzer/checks/test_vcs_check.py index 4f272d49c..d280a8b08 100644 --- a/tests/slsa_analyzer/checks/test_vcs_check.py +++ b/tests/slsa_analyzer/checks/test_vcs_check.py @@ -54,7 +54,6 @@ class TestVCSCheck(MacaronTestCase): def test_vcs_check(self) -> None: """Test the vcs check.""" - check = VCSCheck() git_repo = initiate_repo(REPO_DIR) check_result = CheckResult(justification=[]) # type: ignore diff --git a/tests/slsa_analyzer/ci_service/resources/github/valid2.yaml b/tests/slsa_analyzer/ci_service/resources/github/valid2.yaml index 581b987ea..804f7915d 100644 --- a/tests/slsa_analyzer/ci_service/resources/github/valid2.yaml +++ b/tests/slsa_analyzer/ci_service/resources/github/valid2.yaml @@ -8,24 +8,24 @@ jobs: if: github.repository != 'micronaut-projects/micronaut-project-template' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Set up JDK - uses: actions/setup-java@v3 - with: - distribution: 'adopt' - java-version: '11' - - name: Publish to Sonatype Snapshots - if: success() - env: - SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} - SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GRADLE_ENTERPRISE_ACCESS_KEY }} - GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GRADLE_ENTERPRISE_CACHE_USERNAME }} - GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GRADLE_ENTERPRISE_CACHE_PASSWORD }} - run: ./gradlew publishToSonatype --no-daemon + - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: | + ${{ runner.os }}-gradle- + - name: Set up JDK + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: '11' + - name: Publish to Sonatype Snapshots + if: success() + env: + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GRADLE_ENTERPRISE_ACCESS_KEY }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GRADLE_ENTERPRISE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GRADLE_ENTERPRISE_CACHE_PASSWORD }} + run: ./gradlew publishToSonatype --no-daemon diff --git a/tests/slsa_analyzer/ci_service/test_github_actions.py b/tests/slsa_analyzer/ci_service/test_github_actions.py index b57ea8d9c..0adc3e4a1 100644 --- a/tests/slsa_analyzer/ci_service/test_github_actions.py +++ b/tests/slsa_analyzer/ci_service/test_github_actions.py @@ -8,6 +8,8 @@ import os from pathlib import Path +import pytest + from macaron.code_analyzer.call_graph import CallGraph from macaron.parsers.actionparser import parse as parse_action from macaron.slsa_analyzer.ci_service.github_actions import GHWorkflowType, GitHubActions, GitHubNode @@ -72,6 +74,7 @@ def test_build_call_graph(self) -> None: "GitHubNode(actions/setup-java@v3,GHWorkflowType.EXTERNAL)", ] == [str(node) for node in gh_cg.bfs()] + @pytest.mark.skip() def test_is_detected(self) -> None: """Test detecting GitHub Action config files.""" assert self.github_actions.is_detected(str(self.ga_has_build_kws)) diff --git a/tests/slsa_analyzer/git_service/test_github.py b/tests/slsa_analyzer/git_service/test_github.py index d2d8e75f9..336775c6f 100644 --- a/tests/slsa_analyzer/git_service/test_github.py +++ b/tests/slsa_analyzer/git_service/test_github.py @@ -18,7 +18,6 @@ class TestGitHub(MacaronTestCase): def test_is_detected(self) -> None: """Test the is detected method.""" - github = GitHub() assert github.is_detected("http://github.com/org/name") @@ -32,7 +31,6 @@ def test_is_detected(self) -> None: def test_can_clone_remote_repo(self) -> None: """Test the can clone remote repo method.""" - github = GitHub() with patch.object(GhAPIClient, "get_repo_data", return_value=True): assert github.can_clone_remote_repo("can_clone_repo_url") diff --git a/tests/slsa_analyzer/mock_git_utils.py b/tests/slsa_analyzer/mock_git_utils.py index e606387ee..ec2d286bf 100644 --- a/tests/slsa_analyzer/mock_git_utils.py +++ b/tests/slsa_analyzer/mock_git_utils.py @@ -2,8 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """ -This module contains the methods for preparing mock git repositories for testing -SLSA checks +This module contains the methods for preparing mock git repositories for testing SLSA checks. """ import os diff --git a/tests/slsa_analyzer/runner/test_runner.py b/tests/slsa_analyzer/runner/test_runner.py index fbd642619..e9a9861c5 100644 --- a/tests/slsa_analyzer/runner/test_runner.py +++ b/tests/slsa_analyzer/runner/test_runner.py @@ -15,19 +15,7 @@ class EmptyCheck(BaseCheck): - """An empty check to test the runners. - - Parameters: - ----------- - check_id: str - The id of the check - - should_return: CheckResultType - The result status returned by this check - - parent: list[tuple[str, CheckResultType]] - The list of parent checks that this check depends on. - """ + """An empty check to test the runners.""" def __init__( self, @@ -35,6 +23,19 @@ def __init__( should_return: CheckResultType, parent: list[tuple[str, CheckResultType]], ) -> None: + """Initialize the instance. + + Parameters + ---------- + check_id: str + The id of the check + + should_return: CheckResultType + The result status returned by this check + + parent: list[tuple[str, CheckResultType]] + The list of parent checks that this check depends on. + """ super().__init__(check_id, "This is an empty check.", parent, []) self.should_return = should_return diff --git a/tests/slsa_analyzer/test_analyze_context.py b/tests/slsa_analyzer/test_analyze_context.py index c08580059..0ee965274 100644 --- a/tests/slsa_analyzer/test_analyze_context.py +++ b/tests/slsa_analyzer/test_analyze_context.py @@ -36,7 +36,7 @@ class TestAnalyzeContext(TestCase): def setUp(self) -> None: """ - Setup the sample AnalyzeContext instance + Set up the sample AnalyzeContext instance """ self.analyze_ctx = AnalyzeContext("owner/repo_name", self.MOCK_REPO_PATH, self.MOCK_GIT_OBJ) self.analyze_ctx.ctx_data = self.MOCK_CTX_DATA @@ -78,7 +78,6 @@ def test_gen_create_table_query(self) -> None: """ Test the gen_create_table_query method """ - expect_query = [ "CREATE TABLE IF NOT EXISTS analyze_result " + "(full_name TEXT PRIMARY KEY, branch_name TEXT, commit_sha TEXT, commit_date TEXT, " diff --git a/tests/slsa_analyzer/test_slsa_requirements.py b/tests/slsa_analyzer/test_slsa_requirements.py index e4a9ea4b2..d1ec42ba3 100644 --- a/tests/slsa_analyzer/test_slsa_requirements.py +++ b/tests/slsa_analyzer/test_slsa_requirements.py @@ -41,8 +41,7 @@ def test_status(self) -> None: def test_get_requirements_dict(self) -> None: """ - Test if all the requirements defined in ReqName class are - included in the returned dictionary + Test if all the requirements defined in ReqName class are included in the returned dictionary. """ all_reqs = get_requirements_dict() assert all(req in all_reqs for req in BUILD_REQ_DESC)