From 5890b3734a47cbd72b2e0e6daae2d27068d9968b Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Fri, 10 Jul 2026 12:03:32 +0200 Subject: [PATCH] Improved Flatpak workflow with enhanced artifact validation (#850) --- .github/workflows/build-and-release.yml | 397 +++++++++++++++++++++--- 1 file changed, 348 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 375d86a7..3f366b4e 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -21,6 +21,11 @@ env: RETENTION_RELEASE_ASSETS: 30 FLATPAK_REPOSITORY: MindWorkAI/Flatpak FLATPAK_WORKFLOW: flatpak.yml + FLATPAK_YQ_VERSION: 4.44.6 + FLATPAK_YQ_SHA256: 0c2b24e645b57d8e7c0566d18643a6d4f5580feeea3878127354a46f2a1e4598 + FLATPAK_UV_VERSION: 0.11.28 + FLATPAK_UV_SHA256: e490a6464492183c5d4534a5527fb4440f7f2bb2f228162ad7e4afe076dc0224 + FLATPAK_FREEDESKTOP_VERSION: "25.08" jobs: determine_run_mode: @@ -246,7 +251,6 @@ jobs: flatpak_commit: ${{ steps.sync.outputs.flatpak_commit }} env: - GH_TOKEN: ${{ secrets.FLATPAK_WORKFLOW_TOKEN }} AI_STUDIO_TAG: ${{ needs.read_metadata.outputs.formatted_version }} AI_STUDIO_COMMIT: ${{ needs.read_metadata.outputs.source_commit }} PDFIUM_CHROMIUM_REVISION: ${{ needs.read_metadata.outputs.pdfium_chromium_revision }} @@ -260,18 +264,48 @@ jobs: ref: main path: flatpak fetch-depth: 0 + persist-credentials: false - - name: Install Flatpak sync tools + - name: Install Flatpak sync tools and SDK run: | set -euo pipefail - sudo curl -fsSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64 - sudo chmod +x /usr/local/bin/yq + sudo apt-get update + sudo apt-get install --no-install-recommends -y flatpak - curl -LsSf https://astral.sh/uv/install.sh | sh - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - "$HOME/.local/bin/uv" --version - yq --version + flatpak remote-add \ + --user \ + --if-not-exists \ + flathub \ + https://flathub.org/repo/flathub.flatpakrepo + flatpak install \ + --user \ + --noninteractive \ + -y \ + flathub \ + "org.freedesktop.Sdk//${FLATPAK_FREEDESKTOP_VERSION}" \ + "org.freedesktop.Sdk.Extension.dotnet9//${FLATPAK_FREEDESKTOP_VERSION}" + + tools_dir="$RUNNER_TEMP/flatpak-sync-tools" + mkdir -p "$tools_dir" + + curl -fsSL \ + -o "$tools_dir/yq" \ + "https://github.com/mikefarah/yq/releases/download/v${FLATPAK_YQ_VERSION}/yq_linux_amd64" + echo "${FLATPAK_YQ_SHA256} ${tools_dir}/yq" | sha256sum --check --strict + chmod +x "$tools_dir/yq" + + uv_archive="$RUNNER_TEMP/uv-x86_64-unknown-linux-gnu.tar.gz" + curl -fsSL \ + -o "$uv_archive" \ + "https://github.com/astral-sh/uv/releases/download/${FLATPAK_UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" + echo "${FLATPAK_UV_SHA256} ${uv_archive}" | sha256sum --check --strict + tar -xzf "$uv_archive" -C "$tools_dir" + install -m 0755 "$tools_dir/uv-x86_64-unknown-linux-gnu/uv" "$tools_dir/uv" + + echo "$tools_dir" >> "$GITHUB_PATH" + "$tools_dir/uv" --version + "$tools_dir/yq" --version - name: Update Flatpak release sources working-directory: flatpak @@ -305,11 +339,27 @@ jobs: ' org.MindWorkAI.AIStudio.yml ./update-dependencies + + test "$(yq -r '(.modules[] | select(.name == "mind-work-ai-studio")).sources[0].tag' org.MindWorkAI.AIStudio.yml)" = "$AI_STUDIO_TAG" + test "$(yq -r '(.modules[] | select(.name == "mind-work-ai-studio")).sources[0].commit' org.MindWorkAI.AIStudio.yml)" = "$AI_STUDIO_COMMIT" + test "$(yq -r '(.modules[] | select(.name == "mind-work-ai-studio")).sources[] | select(.type == "archive" and .only-arches[0] == "x86_64").url' org.MindWorkAI.AIStudio.yml)" = "$PDFIUM_X64_URL" + test "$(yq -r '(.modules[] | select(.name == "mind-work-ai-studio")).sources[] | select(.type == "archive" and .only-arches[0] == "x86_64").sha256' org.MindWorkAI.AIStudio.yml)" = "$PDFIUM_X64_SHA256" + test "$(yq -r '(.modules[] | select(.name == "mind-work-ai-studio")).sources[] | select(.type == "archive" and .only-arches[0] == "aarch64").url' org.MindWorkAI.AIStudio.yml)" = "$PDFIUM_ARM64_URL" + test "$(yq -r '(.modules[] | select(.name == "mind-work-ai-studio")).sources[] | select(.type == "archive" and .only-arches[0] == "aarch64").sha256' org.MindWorkAI.AIStudio.yml)" = "$PDFIUM_ARM64_SHA256" + + for generated_source in cargo-sources.json dotnet-sources.json tauri-cli-sources.json; do + test -s "$generated_source" + jq -e 'type == "array" and length > 0' "$generated_source" > /dev/null + done + + git diff --check git diff --stat - name: Commit and merge Flatpak sync id: sync working-directory: flatpak + env: + GH_TOKEN: ${{ secrets.FLATPAK_WORKFLOW_TOKEN }} run: | set -euo pipefail @@ -322,12 +372,23 @@ jobs: if git diff --cached --quiet; then echo "Flatpak repository is already synced for ${AI_STUDIO_TAG}." - flatpak_commit=$(git rev-parse origin/main) + flatpak_commit=$(git rev-parse HEAD) + remote_main=$(git ls-remote origin refs/heads/main | awk '{print $1}') + if [ "$remote_main" != "$flatpak_commit" ]; then + echo "Flatpak main advanced from ${flatpak_commit} to ${remote_main} during synchronization." + exit 1 + fi + echo "flatpak_commit=${flatpak_commit}" >> "$GITHUB_OUTPUT" exit 0 fi git commit -m "Sync AI Studio ${AI_STUDIO_TAG}" + sync_commit=$(git rev-parse HEAD) + + basic_auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 -w 0) + git config --local http.https://github.com/.extraheader "AUTHORIZATION: basic ${basic_auth}" + trap 'git config --local --unset-all http.https://github.com/.extraheader || true' EXIT remote_branch_sha=$(git ls-remote --heads origin "$branch" | awk '{print $1}') if [ -n "$remote_branch_sha" ]; then @@ -353,6 +414,7 @@ jobs: pr_number="${pr_url##*/}" fi + mergeable="UNKNOWN" for attempt in {1..30}; do pr_state=$(gh pr view "$pr_number" --repo "$FLATPAK_REPOSITORY" --json mergeable,mergeStateStatus) mergeable=$(echo "$pr_state" | jq -r '.mergeable') @@ -360,17 +422,54 @@ jobs: echo "PR #${pr_number}: mergeable=${mergeable}, mergeStateStatus=${merge_state_status}" - if [ "$mergeable" != "UNKNOWN" ]; then + if [ "$mergeable" = "MERGEABLE" ]; then break fi + if [ "$mergeable" = "CONFLICTING" ]; then + echo "Flatpak sync PR #${pr_number} has merge conflicts." + exit 1 + fi + sleep 5 done - gh pr merge "$pr_number" --repo "$FLATPAK_REPOSITORY" --merge --delete-branch + if [ "$mergeable" != "MERGEABLE" ]; then + echo "Timed out waiting for Flatpak sync PR #${pr_number} to become mergeable." + exit 1 + fi - flatpak_commit=$(gh api "repos/${FLATPAK_REPOSITORY}/commits/main" --jq .sha) - echo "flatpak_commit=${flatpak_commit}" >> "$GITHUB_OUTPUT" + gh pr merge "$pr_number" \ + --repo "$FLATPAK_REPOSITORY" \ + --merge \ + --delete-branch \ + --match-head-commit "$sync_commit" + + for attempt in {1..120}; do + pr_state=$(gh pr view "$pr_number" \ + --repo "$FLATPAK_REPOSITORY" \ + --json state,mergedAt,mergeCommit) + state=$(echo "$pr_state" | jq -r '.state') + merged_at=$(echo "$pr_state" | jq -r '.mergedAt // empty') + merge_commit=$(echo "$pr_state" | jq -r '.mergeCommit.oid // empty') + + echo "PR #${pr_number}: state=${state}, mergedAt=${merged_at:-pending}, mergeCommit=${merge_commit:-pending}" + + if [ -n "$merged_at" ] && [ -n "$merge_commit" ]; then + echo "flatpak_commit=${merge_commit}" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "$state" = "CLOSED" ]; then + echo "Flatpak sync PR #${pr_number} was closed without being merged." + exit 1 + fi + + sleep 5 + done + + echo "Timed out waiting for Flatpak sync PR #${pr_number} to be merged." + exit 1 collect_flatpak_artifacts: name: Collect Flatpak artifacts @@ -381,63 +480,233 @@ jobs: contents: read env: - GH_TOKEN: ${{ secrets.FLATPAK_WORKFLOW_TOKEN }} FLATPAK_COMMIT: ${{ needs.sync_flatpak_repo.outputs.flatpak_commit }} steps: - name: Wait for Flatpak main build id: flatpak_run + env: + GH_TOKEN: ${{ secrets.FLATPAK_WORKFLOW_TOKEN }} run: | set -euo pipefail - run_id="" + find_run_id() { + local created_after="${1:-}" + local runs - for attempt in {1..180}; do - run_json=$(gh run list \ + runs=$(gh run list \ --repo "$FLATPAK_REPOSITORY" \ --workflow "$FLATPAK_WORKFLOW" \ --branch main \ --commit "$FLATPAK_COMMIT" \ - --event push \ - --limit 1 \ - --json databaseId,status,conclusion,url,headSha) + --limit 20 \ + --json databaseId,event,headSha,createdAt) - run_id=$(echo "$run_json" | jq -r '.[0].databaseId // empty') - - if [ -n "$run_id" ]; then - status=$(echo "$run_json" | jq -r '.[0].status') - conclusion=$(echo "$run_json" | jq -r '.[0].conclusion') - url=$(echo "$run_json" | jq -r '.[0].url') - - echo "Flatpak run ${run_id}: status=${status}, conclusion=${conclusion}, url=${url}" - - if [ "$status" = "completed" ]; then - if [ "$conclusion" != "success" ]; then - echo "Flatpak workflow failed with conclusion '${conclusion}'." - exit 1 - fi - - echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" - exit 0 - fi + if [ -n "$created_after" ]; then + echo "$runs" | jq -r \ + --arg commit "$FLATPAK_COMMIT" \ + --arg created_after "$created_after" \ + '[.[] | select(.headSha == $commit and .event == "workflow_dispatch" and .createdAt >= $created_after)][0].databaseId // empty' else - echo "Waiting for Flatpak workflow on commit ${FLATPAK_COMMIT}..." + echo "$runs" | jq -r \ + --arg commit "$FLATPAK_COMMIT" \ + '[.[] | select(.headSha == $commit and (.event == "push" or .event == "workflow_dispatch"))][0].databaseId // empty' + fi + } + + validate_required_artifacts() { + local run_id="$1" + local artifacts + local expected_name + local match_count + local custom_count + + artifacts=$(gh api "repos/${FLATPAK_REPOSITORY}/actions/runs/${run_id}/artifacts?per_page=100") + custom_count=$(echo "$artifacts" | jq \ + '[.artifacts[] | select(.name | startswith("MindWork AI Studio Flatpak ("))] | length') + + if [ "$custom_count" -ne 2 ]; then + echo "Flatpak run ${run_id} contains ${custom_count} release artifacts; expected 2." + return 1 fi + for expected_name in \ + "MindWork AI Studio Flatpak (x86_64)" \ + "MindWork AI Studio Flatpak (aarch64)"; do + match_count=$(echo "$artifacts" | jq \ + --arg name "$expected_name" \ + '[.artifacts[] | select(.name == $name and .expired == false)] | length') + + if [ "$match_count" -ne 1 ]; then + echo "Flatpak run ${run_id} does not contain one active '${expected_name}' artifact." + return 1 + fi + done + } + + wait_for_run() { + local run_id="$1" + local run_json + local status + local conclusion + local head_sha + local url + + for attempt in {1..180}; do + run_json=$(gh run view "$run_id" \ + --repo "$FLATPAK_REPOSITORY" \ + --json status,conclusion,headSha,url) + status=$(echo "$run_json" | jq -r '.status') + conclusion=$(echo "$run_json" | jq -r '.conclusion // empty') + head_sha=$(echo "$run_json" | jq -r '.headSha') + url=$(echo "$run_json" | jq -r '.url') + + echo "Flatpak run ${run_id}: status=${status}, conclusion=${conclusion:-pending}, url=${url}" + + if [ "$head_sha" != "$FLATPAK_COMMIT" ]; then + echo "Flatpak run ${run_id} targets '${head_sha}', expected '${FLATPAK_COMMIT}'." + return 1 + fi + + if [ "$status" = "completed" ]; then + if [ "$conclusion" = "success" ] && validate_required_artifacts "$run_id"; then + return 0 + fi + + echo "Flatpak run ${run_id} completed without usable release artifacts." + return 1 + fi + + sleep 20 + done + + echo "Timed out waiting for Flatpak run ${run_id}." + return 2 + } + + run_id="" + for attempt in {1..15}; do + run_id=$(find_run_id) + if [ -n "$run_id" ]; then + break + fi + + echo "Waiting for Flatpak workflow on commit ${FLATPAK_COMMIT}..." sleep 20 done - echo "Timed out waiting for the Flatpak workflow on commit ${FLATPAK_COMMIT}." - exit 1 + if [ -z "$run_id" ]; then + current_main=$(gh api "repos/${FLATPAK_REPOSITORY}/commits/main" --jq .sha) + if [ "$current_main" != "$FLATPAK_COMMIT" ]; then + echo "No Flatpak run exists for ${FLATPAK_COMMIT}, and Flatpak main has advanced to ${current_main}." + exit 1 + fi + + dispatch_started_at=$(date --utc +'%Y-%m-%dT%H:%M:%SZ') + gh workflow run "$FLATPAK_WORKFLOW" \ + --repo "$FLATPAK_REPOSITORY" \ + --ref main \ + -f "artifact_retention_days=${RETENTION_INTERMEDIATE_ASSETS}" + + for attempt in {1..15}; do + run_id=$(find_run_id "$dispatch_started_at") + if [ -n "$run_id" ]; then + break + fi + + echo "Waiting for the dispatched Flatpak workflow on commit ${FLATPAK_COMMIT}..." + sleep 20 + done + fi + + if [ -z "$run_id" ]; then + echo "Timed out waiting for a Flatpak workflow to start on commit ${FLATPAK_COMMIT}." + exit 1 + fi + + set +e + wait_for_run "$run_id" + wait_result=$? + set -e + + if [ "$wait_result" -eq 0 ]; then + echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "$wait_result" -eq 2 ]; then + exit 1 + fi + + echo "Re-running Flatpak run ${run_id} once." + previous_attempt=$(gh api "repos/${FLATPAK_REPOSITORY}/actions/runs/${run_id}" --jq .run_attempt) + gh run rerun "$run_id" --repo "$FLATPAK_REPOSITORY" + + rerun_started=false + for attempt in {1..60}; do + rerun_state=$(gh api "repos/${FLATPAK_REPOSITORY}/actions/runs/${run_id}") + current_attempt=$(echo "$rerun_state" | jq -r '.run_attempt') + current_status=$(echo "$rerun_state" | jq -r '.status') + + if [ "$current_attempt" -gt "$previous_attempt" ] || [ "$current_status" != "completed" ]; then + rerun_started=true + break + fi + + sleep 2 + done + + if [ "$rerun_started" != "true" ]; then + echo "Timed out waiting for Flatpak run ${run_id} to start its retry." + exit 1 + fi + + if ! wait_for_run "$run_id"; then + echo "Flatpak run ${run_id} did not succeed after one retry." + exit 1 + fi + + echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" - name: Download Flatpak artifacts env: + GH_TOKEN: ${{ secrets.FLATPAK_WORKFLOW_TOKEN }} RUN_ID: ${{ steps.flatpak_run.outputs.run_id }} run: | set -euo pipefail mkdir -p flatpak-artifacts - gh run download "$RUN_ID" --repo "$FLATPAK_REPOSITORY" --dir flatpak-artifacts + gh run download "$RUN_ID" \ + --repo "$FLATPAK_REPOSITORY" \ + --name "MindWork AI Studio Flatpak (x86_64)" \ + --name "MindWork AI Studio Flatpak (aarch64)" \ + --dir flatpak-artifacts + + expected_files=( + "MindWork AI Studio_x86_64.flatpak" + "MindWork AI Studio Plugin Pandoc_x86_64.flatpak" + "MindWork AI Studio_aarch64.flatpak" + "MindWork AI Studio Plugin Pandoc_aarch64.flatpak" + ) + + flatpak_count=$(find flatpak-artifacts -type f -name '*.flatpak' | wc -l | tr -d ' ') + if [ "$flatpak_count" -ne 4 ]; then + echo "Downloaded ${flatpak_count} Flatpak files; expected 4." + find flatpak-artifacts -type f -print + exit 1 + fi + + for expected_file in "${expected_files[@]}"; do + match_count=$(find flatpak-artifacts -type f -name "$expected_file" | wc -l | tr -d ' ') + if [ "$match_count" -ne 1 ]; then + echo "Expected exactly one '${expected_file}', found ${match_count}." + exit 1 + fi + + matched_file=$(find flatpak-artifacts -type f -name "$expected_file" -print -quit) + test -s "$matched_file" + done + find flatpak-artifacts -type f -name '*.flatpak' -print - name: Upload Flatpak artifacts @@ -446,6 +715,7 @@ jobs: name: MindWork AI Studio Flatpak Release path: flatpak-artifacts/**/*.flatpak if-no-files-found: error + overwrite: true retention-days: ${{ env.RETENTION_INTERMEDIATE_ASSETS }} build_main: @@ -1032,16 +1302,19 @@ jobs: - name: Prepare release assets env: VERSION: ${{ needs.read_metadata.outputs.version }} - + run: | + set -euo pipefail + RELEASE_DIR="$GITHUB_WORKSPACE/release/assets" - + declare -A release_asset_sources + # Ensure the release directory exists: mkdir -p "$RELEASE_DIR" - + # Find and process files in the artifacts directory: - find "$GITHUB_WORKSPACE/artifacts" -type f | while read -r FILE; do - + while IFS= read -r -d '' FILE; do + if [[ "$FILE" == *"osx-x64"* && "$FILE" == *".tar.gz.sig" ]]; then TARGET_NAME="MindWork AI Studio_x64.app.tar.gz.sig" elif [[ "$FILE" == *"osx-x64"* && "$FILE" == *".tar.gz" ]]; then @@ -1054,10 +1327,36 @@ jobs: TARGET_NAME="$(basename "$FILE")" TARGET_NAME=$(echo "$TARGET_NAME" | sed "s/_${VERSION}//") fi - + + if [ -n "${release_asset_sources[$TARGET_NAME]+x}" ]; then + echo "Duplicate release asset name '${TARGET_NAME}':" + echo " ${release_asset_sources[$TARGET_NAME]}" + echo " ${FILE}" + exit 1 + fi + + release_asset_sources[$TARGET_NAME]="$FILE" + cp "$FILE" "${RELEASE_DIR}/${TARGET_NAME}" + done < <(find "$GITHUB_WORKSPACE/artifacts" -type f -print0) + + expected_flatpaks=( + "MindWork AI Studio_x86_64.flatpak" + "MindWork AI Studio Plugin Pandoc_x86_64.flatpak" + "MindWork AI Studio_aarch64.flatpak" + "MindWork AI Studio Plugin Pandoc_aarch64.flatpak" + ) + + flatpak_count=$(find "$RELEASE_DIR" -maxdepth 1 -type f -name '*.flatpak' | wc -l | tr -d ' ') + if [ "$flatpak_count" -ne 4 ]; then + echo "Prepared ${flatpak_count} Flatpak release assets; expected 4." + exit 1 + fi + + for expected_flatpak in "${expected_flatpaks[@]}"; do + test -s "${RELEASE_DIR}/${expected_flatpak}" done - + # Display the structure of the release directory: ls -Rlhat $GITHUB_WORKSPACE/release/assets