👷 Update LLM translation CI, add language matrix and extra commands, prepare for scheduled run (#14529)

.github/workflows/translate.yml

@@ -16,7 +16,7 @@ on:
           - update-outdated
           - add-missing
           - update-and-add
-          - remove-all-removable
+          - remove-removable
       language:
         description: Language to translate to as a letter code (e.g. "es" for Spanish)
         type: string
@@ -32,9 +32,42 @@ env:
   UV_SYSTEM_PYTHON: 1
 
 jobs:
-  job:
+  langs:
-    if: github.repository_owner == 'fastapi'
     runs-on: ubuntu-latest
+    outputs:
+      langs: ${{ steps.show-langs.outputs.langs }}
+      commands: ${{ steps.show-langs.outputs.commands }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          cache-dependency-glob: |
+            requirements**.txt
+            pyproject.toml
+      - name: Install Dependencies
+        run: uv pip install -r requirements-github-actions.txt -r requirements-translations.txt
+      - name: Export Language Codes
+        id: show-langs
+        run: |
+          echo "langs=$(python ./scripts/translate.py llm-translatable-json)" >> $GITHUB_OUTPUT
+          echo "commands=$(python ./scripts/translate.py commands-json)" >> $GITHUB_OUTPUT
+        env:
+          LANGUAGE: ${{ github.event.inputs.language }}
+          COMMAND: ${{ github.event.inputs.command }}
+
+  translate:
+    if: github.repository_owner == 'fastapi'
+    needs: langs
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        lang: ${{ fromJson(needs.langs.outputs.langs) }}
+        command: ${{ fromJson(needs.langs.outputs.commands) }}
     permissions:
       contents: write
     steps:
@@ -50,8 +83,6 @@ jobs:
       - name: Setup uv
         uses: astral-sh/setup-uv@v7
         with:
-          version: "0.4.15"
-          enable-cache: true
           cache-dependency-glob: |
             requirements**.txt
             pyproject.toml
@@ -68,10 +99,11 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       - name: FastAPI Translate
         run: |
-          python ./scripts/translate.py ${{ github.event.inputs.command }}
+          python ./scripts/translate.py ${{ matrix.command }}
           python ./scripts/translate.py make-pr
         env:
           GITHUB_TOKEN: ${{ secrets.FASTAPI_TRANSLATIONS }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          LANGUAGE: ${{ github.event.inputs.language }}
+          LANGUAGE: ${{ matrix.lang }}
           EN_PATH: ${{ github.event.inputs.en_path }}
+          COMMAND: ${{ matrix.command }}

scripts/translate.py

@@ -1,3 +1,4 @@
+import json
 import secrets
 import subprocess
 from collections.abc import Iterable
@@ -828,6 +829,65 @@ def translate_lang(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) ->
         print(f"Done translating: {p}")
 
 
+def get_llm_translatable() -> list[str]:
+    translatable_langs = []
+    langs = get_langs()
+    for lang in langs:
+        if lang == "en":
+            continue
+        lang_prompt_path = Path(f"docs/{lang}/llm-prompt.md")
+        if lang_prompt_path.exists():
+            translatable_langs.append(lang)
+    return translatable_langs
+
+
+@app.command()
+def list_llm_translatable() -> list[str]:
+    translatable_langs = get_llm_translatable()
+    print("LLM translatable languages:", translatable_langs)
+    return translatable_langs
+
+
+@app.command()
+def llm_translatable_json(
+    language: Annotated[str | None, typer.Option(envvar="LANGUAGE")] = None,
+) -> None:
+    translatable_langs = get_llm_translatable()
+    if language:
+        if language in translatable_langs:
+            print(json.dumps([language]))
+            return
+        else:
+            raise typer.Exit(code=1)
+    print(json.dumps(translatable_langs))
+
+
+@app.command()
+def commands_json(
+    command: Annotated[str | None, typer.Option(envvar="COMMAND")] = None,
+) -> None:
+    available_commands = [
+        "translate-page",
+        "translate-lang",
+        "update-outdated",
+        "add-missing",
+        "update-and-add",
+        "remove-removable",
+    ]
+    default_commands = [
+        "remove-removable",
+        "update-outdated",
+        "add-missing",
+    ]
+    if command:
+        if command in available_commands:
+            print(json.dumps([command]))
+            return
+        else:
+            raise typer.Exit(code=1)
+    print(json.dumps(default_commands))
+
+
 @app.command()
 def list_removable(language: str) -> list[Path]:
     removable_paths: list[Path] = []
@@ -939,6 +999,7 @@ def update_and_add(language: Annotated[str, typer.Option(envvar="LANGUAGE")]) ->
 def make_pr(
     *,
     language: Annotated[str | None, typer.Option(envvar="LANGUAGE")] = None,
+    command: Annotated[str | None, typer.Option(envvar="COMMAND")] = None,
     github_token: Annotated[str, typer.Option(envvar="GITHUB_TOKEN")],
     github_repository: Annotated[str, typer.Option(envvar="GITHUB_REPOSITORY")],
 ) -> None:
@@ -955,6 +1016,8 @@ def make_pr(
     branch_name = "translate"
     if language:
         branch_name += f"-{language}"
+    if command:
+        branch_name += f"-{command}"
     branch_name += f"-{secrets.token_hex(4)}"
     print(f"Creating a new branch {branch_name}")
     subprocess.run(["git", "checkout", "-b", branch_name], check=True)
@@ -965,6 +1028,8 @@ def make_pr(
     message = "🌐 Update translations"
     if language:
         message += f" for {language}"
+    if command:
+        message += f" ({command})"
     subprocess.run(["git", "commit", "-m", message], check=True)
     print("Pushing branch")
     subprocess.run(["git", "push", "origin", branch_name], check=True)