👷 Switch from Codecov to Smokeshow plus pytest-cov to pure coverage for internal tests (#5583)

.github/workflows/smokeshow.yml

@@ -0,0 +1,35 @@
+name: Smokeshow
+
+on:
+  workflow_run:
+    workflows: [Test]
+    types: [completed]
+
+permissions:
+  statuses: write
+
+jobs:
+  smokeshow:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - run: pip install smokeshow
+
+      - uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: test.yml
+          commit: ${{ github.event.workflow_run.head_sha }}
+
+      - run: smokeshow upload coverage-html
+        env:
+          SMOKESHOW_GITHUB_STATUS_DESCRIPTION: Coverage {coverage-percentage}
+          SMOKESHOW_GITHUB_COVERAGE_THRESHOLD: 100
+          SMOKESHOW_GITHUB_CONTEXT: coverage
+          SMOKESHOW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SMOKESHOW_GITHUB_PR_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+          SMOKESHOW_AUTH_KEY: ${{ secrets.SMOKESHOW_AUTH_KEY }}

.github/workflows/test.yml

@@ -31,7 +31,43 @@ jobs:
         run: pip install -e .[all,dev,doc,test]
       - name: Lint
         run: bash scripts/lint.sh
+      - run: mkdir coverage
       - name: Test
         run: bash scripts/test.sh
-      - name: Upload coverage
-        uses: codecov/codecov-action@v3
+        env:
+          COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}
+          CONTEXT: ${{ runner.os }}-py${{ matrix.python-version }}
+      - name: Store coverage files
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: coverage
+  coverage-combine:
+    needs: [test]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Get coverage files
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage
+          path: coverage
+
+      - run: pip install coverage[toml]
+
+      - run: ls -la coverage
+      - run: coverage combine coverage
+      - run: coverage report
+      - run: coverage html --show-contexts --title "Coverage for ${{ github.sha }}"
+
+      - name: Store coverage HTML
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-html
+          path: htmlcov

pyproject.toml

@@ -50,7 +50,7 @@ Documentation = "https://fastapi.tiangolo.com/"
 [project.optional-dependencies]
 test = [
     "pytest >=7.1.3,<8.0.0",
-    "pytest-cov >=2.12.0,<5.0.0",
+    "coverage[toml] >= 6.5.0,<7.0",
     "mypy ==0.982",
     "flake8 >=3.8.3,<6.0.0",
     "black == 22.8.0",
@@ -147,3 +147,12 @@ filterwarnings = [
     # TODO remove pytest-cov
     'ignore::pytest.PytestDeprecationWarning:pytest_cov',
 ]
+
+[tool.coverage.run]
+parallel = true
+source = [
+    "docs_src",
+    "tests",
+    "fastapi"
+]
+context = '${CONTEXT}'

scripts/test-cov-html.sh

@@ -3,4 +3,7 @@
 set -e
 set -x
 
-bash scripts/test.sh --cov-report=html ${@}
+bash scripts/test.sh ${@}
+coverage combine
+coverage report --show-missing
+coverage html

scripts/test.sh

@@ -6,4 +6,4 @@ set -x
 # Check README.md is up to date
 python ./scripts/docs.py verify-readme
 export PYTHONPATH=./docs_src
-pytest --cov=fastapi --cov=tests --cov=docs_src --cov-report=term-missing:skip-covered --cov-report=xml tests ${@}
+coverage run -m pytest tests ${@}