From 5a38b3be715b8c2111f0d59f405e2aaeccdcc7b1 Mon Sep 17 00:00:00 2001 From: ShotaKitazawa Date: Sun, 14 Apr 2024 21:53:17 +0900 Subject: [PATCH 1/6] separate build Action --- .../workflows/{build.yml => build-branch.yml} | 8 +-- .github/workflows/build-tag.yml | 49 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) rename .github/workflows/{build.yml => build-branch.yml} (92%) create mode 100644 .github/workflows/build-tag.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build-branch.yml similarity index 92% rename from .github/workflows/build.yml rename to .github/workflows/build-branch.yml index 0dffbce..05a0e01 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build-branch.yml @@ -1,6 +1,8 @@ -name: build dreamkast-weaver image +name: build dreamkast-weaver image when branches are pushed -on: push +on: + push: + branches: ["*"] jobs: build: @@ -18,7 +20,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ap-northeast-1 + aws-region: us-west-2 - name: Login to Amazon ECR id: login-ecr diff --git a/.github/workflows/build-tag.yml b/.github/workflows/build-tag.yml new file mode 100644 index 0000000..a33d907 --- /dev/null +++ b/.github/workflows/build-tag.yml @@ -0,0 +1,49 @@ +name: build dreamkast-weaver image when tags are pushed + +on: + push: + tags: ["*"] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ap-northeast-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ steps.login-ecr.outputs.registry }}/dreamkast-weaver + tags: | + type=sha,prefix=,format=long + type=ref,event=tag + + - name: Build + id: docker_build + uses: docker/build-push-action@v5 + with: + context: ./ + file: Dockerfile + builder: ${{ steps.buildx.outputs.name }} + push: true + provenance: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max From 618602d9377f36a9c36a57e230117dfccae08a75 Mon Sep 17 00:00:00 2001 From: shota-kitazawa Date: Fri, 3 May 2024 21:57:09 +0900 Subject: [PATCH 2/6] rename reviewapp.yml > add-labels.yml --- .github/workflows/add-labels.yml | 54 +++++++++++++++++++++ .github/workflows/reviewapps.yml | 82 -------------------------------- 2 files changed, 54 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/add-labels.yml delete mode 100644 .github/workflows/reviewapps.yml diff --git a/.github/workflows/add-labels.yml b/.github/workflows/add-labels.yml new file mode 100644 index 0000000..933cec5 --- /dev/null +++ b/.github/workflows/add-labels.yml @@ -0,0 +1,54 @@ +name: add labels for some cases + +on: pull_request + +jobs: + reviewapps: + name: grant 'reviewapps' label if there are any changes in PR's source code. + runs-on: ubuntu-latest # windows-latest | macos-latest + if: ${{ ! startsWith(github.head_ref, 'renovate/') }} + steps: + - uses: actions/checkout@v4 + + - name: Get changed files + id: check-paths-ignore + uses: tj-actions/changed-files@v42 + with: + files_ignore: | + .github/**/*.yml + **.md + + # to trigger other Actions caused by adding reviewapp Label + - name: Generate token + if: steps.check-paths-ignore.outputs.any_changed == 'true' + id: generate_token + uses: tibdex/github-app-token@v2 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.PRIVATE_KEY }} + + - name: Labeling 'reviewapps' to PR + if: steps.check-paths-ignore.outputs.any_changed == 'true' + uses: actions/github-script@v7 + with: + github-token: ${{ steps.generate_token.outputs.token }} + result-encoding: string + script: | + const targetLabel = 'reviewapps'; + issue = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + flag = false; + issue.data.labels.filter(label => { + if (label.name == targetLabel) { flag = true; }; + }); + if (!flag) { + github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [targetLabel] + }); + } diff --git a/.github/workflows/reviewapps.yml b/.github/workflows/reviewapps.yml deleted file mode 100644 index d393c7c..0000000 --- a/.github/workflows/reviewapps.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: grant 'reviewapps' label if there are any changes in PR's source code. - -on: pull_request - -jobs: - labeling: - runs-on: ubuntu-latest # windows-latest | macos-latest - name: grant 'reviewapps' label - if: ${{ ! startsWith(github.head_ref, 'renovate/') }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v40 - with: - use_fork_point: true - - - name: List all changed files - id: check-paths-ignore - env: - ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" - run: | - FLAG=$(cat << '_EOF_' | python - import os - import sys - import pathlib - paths_ignore = [ - '.github/**/*.yml', - '**.md', - ] - all_changed_files = os.getenv("ALL_CHANGED_FILES").split() - for filename in all_changed_files: - if not any(list(map(lambda pattern: pathlib.PurePath(filename).match(pattern), paths_ignore))): - print("false") - sys.exit() - print("true") - _EOF_ - ) - echo "FLAG=${FLAG}" >> $GITHUB_OUTPUT - - - name: Labeling 'reviewapps' to PR - uses: actions/github-script@v6 - id: set-result - if: ${{ steps.check-paths-ignore.outputs.FLAG == 'false' }} - with: - result-encoding: string - script: | - const message = ` - Review app - * https://dreamkast-weaver-%d.dev.cloudnativedays.jp - `; - const targetLabel = 'reviewapps'; - issue = await github.rest.issues.get({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - flag = false; - issue.data.labels.filter(label => { - if (label.name == targetLabel) { flag = true; }; - }); - if (!flag) { - github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - labels: [targetLabel] - }); - github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: message.replace("%d", context.issue.number), - }); - } From f2b9f8b4e1498f666cf81ad31f92a4df9c09b126 Mon Sep 17 00:00:00 2001 From: shota-kitazawa Date: Fri, 3 May 2024 21:58:28 +0900 Subject: [PATCH 3/6] create reviewapp Action --- .github/workflows/reviewapp.yml | 131 ++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 .github/workflows/reviewapp.yml diff --git a/.github/workflows/reviewapp.yml b/.github/workflows/reviewapp.yml new file mode 100644 index 0000000..233ee21 --- /dev/null +++ b/.github/workflows/reviewapp.yml @@ -0,0 +1,131 @@ +name: deploy reviewapp + +on: + push: + branches-ignore: + - main + pull_request: + types: + - labeled + +# this workflow should not work with cleanup workflow +concurrency: reviewapps + +jobs: + reviewapp: + name: create reviewapp files to dreamkast-infra + runs-on: ubuntu-latest + permissions: + pull-requests: write + if: ${{ ! startsWith(github.head_ref, 'renovate/') }} + steps: + - uses: actions/checkout@v4 + with: + # checout branch even if triggered by PullRequest Label + ref: ${{ github.head_ref }} + + - id: check_if_reviewapp + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + PR_NUMBER=$(gh pr view --json number -q .number) + gh pr view --json labels -q .labels | jq '.[] | select(.name == "reviewapps")' -e + echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" + # the following is needed because it cannot be retrieved + # from ${{ github.sha }} if triggered by PullRequest Label + echo "pr_commit=$(gh pr view --json commits | jq -r '.commits[-1].oid')" >> "$GITHUB_OUTPUT" + + - name: Install Go + if: steps.check_if_reviewapp.outcome == 'success' + uses: actions/setup-go@v5 + + - name: Install jsonnet + if: steps.check_if_reviewapp.outcome == 'success' + run: | + go install github.com/google/go-jsonnet/cmd/jsonnet@latest + go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest + + - name: Generate token + if: steps.check_if_reviewapp.outcome == 'success' + id: generate_token + uses: tibdex/github-app-token@v2 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.PRIVATE_KEY }} + + - name: Configure AWS Credentials + if: steps.check_if_reviewapp.outcome == 'success' + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Checkout dreamkast-infra + if: steps.check_if_reviewapp.outcome == 'success' + uses: actions/checkout@v4 + with: + repository: cloudnativedaysjp/dreamkast-infra + path: dreamkast-infra + token: ${{ steps.generate_token.outputs.token }} + + - name: Create reviewapps + if: steps.check_if_reviewapp.outcome == 'success' + working-directory: dreamkast-infra/ + run: ecspresso/reviewapps/build-reviewapp.sh + env: + REPOSITORY_NAME: ${{ github.repository }} + PR_NUMBER: ${{ steps.check_if_reviewapp.outputs.pr_number }} + IMAGE_TAG: ${{ steps.check_if_reviewapp.outputs.pr_commit }} + + - name: Commit files + id: commit_files + if: steps.check_if_reviewapp.outcome == 'success' + continue-on-error: true + working-directory: dreamkast-infra/ + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git status + git add -A + git commit -a -F- < Date: Fri, 3 May 2024 21:59:01 +0900 Subject: [PATCH 4/6] create reviewapp-cleanup Action --- .github/workflows/reviewapp-cleanup.yml | 91 +++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/reviewapp-cleanup.yml diff --git a/.github/workflows/reviewapp-cleanup.yml b/.github/workflows/reviewapp-cleanup.yml new file mode 100644 index 0000000..11e12df --- /dev/null +++ b/.github/workflows/reviewapp-cleanup.yml @@ -0,0 +1,91 @@ +name: cleanup reviewapps + +on: + schedule: + - cron: '*/30 * * * *' + +# this workflow should not work with cleanup workflow +concurrency: reviewapps + +jobs: + cleanup: + name: create reviewapp files to dreamkast-infra + runs-on: ubuntu-latest + steps: + - name: Generate token + id: generate_token + uses: tibdex/github-app-token@v2 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.PRIVATE_KEY }} + + - name: Checkout dreamkast-infra + uses: actions/checkout@v4 + with: + repository: cloudnativedaysjp/dreamkast-infra + path: dreamkast-infra + token: ${{ steps.generate_token.outputs.token }} + + - name: Install ecspresso + uses: kayac/ecspresso@v2 + + - name: Cleanup + working-directory: dreamkast-infra/ecspresso/reviewapps + env: + GH_TOKEN: ${{ github.token }} + PREFIX: weaver- + shell: bash -x {0} + run: | + EXPECTED="$(gh pr list --repo ${{ github.repository }} --label 'reviewapps' --json number --jq '.[].number' | sed 's/^\(.*\)$/'$PREFIX'\1/g')" + ACTUAL="$(ls | grep -E ^$PREFIX)" + # If present only in ACTUAL, cleanup + for act in $ACTUAL; do + EXIST=false + for exp in $EXPECTED; do + [ "$act" = "$exp" ] && EXIST=true + done + [ $EXIST = true ] && continue + bash -x $act/cleanup.sh + rm -rf $act + done + - name: Commit files + id: commit_files + continue-on-error: true + working-directory: dreamkast-infra/ + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git status + git add -A + git commit -a -F- < Date: Fri, 3 May 2024 23:02:35 +0900 Subject: [PATCH 5/6] gitops-stg for ECS --- .github/workflows/gitops-stg.yml | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/.github/workflows/gitops-stg.yml b/.github/workflows/gitops-stg.yml index 6b640fc..2fce639 100644 --- a/.github/workflows/gitops-stg.yml +++ b/.github/workflows/gitops-stg.yml @@ -78,3 +78,84 @@ jobs: pull_number: pr.data.number, merge_method: "squash", }); + + ecs: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v5 + + - name: Install jsonnet + run: | + go install github.com/google/go-jsonnet/cmd/jsonnet@latest + go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest + + - name: Generate token + id: generate_token + uses: tibdex/github-app-token@v2 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.PRIVATE_KEY }} + + - name: Checkout dreamkast-infra + uses: actions/checkout@v4 + with: + repository: cloudnativedaysjp/dreamkast-infra + path: dreamkast-infra + token: ${{ steps.generate_token.outputs.token }} + + - name: Update image-tags + working-directory: dreamkast-infra/ecspresso/stg + run: | + cat << _EOL_ | jsonnet - > ./const.libsonnet.tmp + local const = import './const.libsonnet'; + + const + { + imageTags: const.imageTags + { + dreamkast_weaver: "${{ github.sha }}", + }, + } + _EOL_ + mv const.libsonnet.tmp const.libsonnet + jsonnetfmt -i const.libsonnet + + - name: Commit files + id: commit_files + continue-on-error: true + working-directory: dreamkast-infra/ + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git status + git add -A + git commit -am "Bump docker tag (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA})" + + - name: Push changes + if: steps.commit_files.outcome == 'success' + uses: ad-m/github-push-action@master + with: + github_token: ${{ steps.generate_token.outputs.token }} + repository: cloudnativedaysjp/dreamkast-infra + directory: dreamkast-infra + branch: staging/weaver-main + + - name: Create and Merge Pull Request + if: steps.commit_files.outcome == 'success' + uses: "actions/github-script@v7" + with: + github-token: ${{ steps.generate_token.outputs.token }} + script: | + const pr = await github.rest.pulls.create({ + owner: "cloudnativedaysjp", + repo: "dreamkast-infra", + title: "Automated PR (staging/weaver-main)", + body: "**this PR is automatically created & merged**", + head: "staging/weaver-main", + base: "main" + }); + await github.rest.pulls.merge({ + owner: "cloudnativedaysjp", + repo: "dreamkast-infra", + pull_number: pr.data.number, + merge_method: "squash", + }); From cd5a434cb0362d401da6aa23b8d5a1777f184246 Mon Sep 17 00:00:00 2001 From: shota-kitazawa Date: Sun, 5 May 2024 16:51:48 +0900 Subject: [PATCH 6/6] use actions/labeler Action instead of tj-actions/changed-files & actions/github-script --- .github/labeler.yml | 5 ++++ .github/workflows/add-labels.yml | 41 ++++---------------------------- 2 files changed, 9 insertions(+), 37 deletions(-) create mode 100644 .github/labeler.yml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000..3c8cfd9 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,5 @@ +reviewapps: + - changed-files: + - all-globs-to-any-file: + - '!.github/**/*.yml' + - '!**/*.md' diff --git a/.github/workflows/add-labels.yml b/.github/workflows/add-labels.yml index 933cec5..ed85256 100644 --- a/.github/workflows/add-labels.yml +++ b/.github/workflows/add-labels.yml @@ -3,52 +3,19 @@ name: add labels for some cases on: pull_request jobs: - reviewapps: - name: grant 'reviewapps' label if there are any changes in PR's source code. + labeler: runs-on: ubuntu-latest # windows-latest | macos-latest if: ${{ ! startsWith(github.head_ref, 'renovate/') }} steps: - - uses: actions/checkout@v4 - - - name: Get changed files - id: check-paths-ignore - uses: tj-actions/changed-files@v42 - with: - files_ignore: | - .github/**/*.yml - **.md - # to trigger other Actions caused by adding reviewapp Label - name: Generate token - if: steps.check-paths-ignore.outputs.any_changed == 'true' id: generate_token uses: tibdex/github-app-token@v2 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.PRIVATE_KEY }} - - name: Labeling 'reviewapps' to PR - if: steps.check-paths-ignore.outputs.any_changed == 'true' - uses: actions/github-script@v7 + - id: label-the-PR + uses: actions/labeler@v5 with: - github-token: ${{ steps.generate_token.outputs.token }} - result-encoding: string - script: | - const targetLabel = 'reviewapps'; - issue = await github.rest.issues.get({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - flag = false; - issue.data.labels.filter(label => { - if (label.name == targetLabel) { flag = true; }; - }); - if (!flag) { - github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - labels: [targetLabel] - }); - } + repo-token: ${{ steps.generate_token.outputs.token }}