diff --git a/.github/workflows/priv_release.yml b/.github/workflows/priv_release.yml new file mode 100644 index 00000000000..c9cec2b6ce0 --- /dev/null +++ b/.github/workflows/priv_release.yml @@ -0,0 +1,36 @@ +name: Priv Release + +on: + push: + branches: + - priv-release + schedule: + # beijing: 12 pm. + # utc: 4 am. + - cron: "0 4 * * *" + workflow_dispatch: + inputs: + priv_branch: + required: false + default: "main" + +concurrency: + group: priv-release-${{ github.ref }} + cancel-in-progress: true + +jobs: + release: + name: Release pip + permissions: + contents: read + pull-requests: write + uses: Oneflow-Inc/oneflow/.github/workflows/release.yml@master + with: + is_priv: true + branch: ${{ inputs.priv_branch || 'main' }} + secrets: + ONEFLOW_PRIV_ORG: ${{ secrets.ONEFLOW_PRIV_ORG }} + ONEFLOW_PRIV_GH_TOKEN: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }} + ONEFLOW_PRIV_OSS_BUCKET: ${{ secrets.ONEFLOW_PRIV_OSS_BUCKET }} + OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} + OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d2f4a56566f..ab1aa3ceae2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,26 @@ on: placeholder: description: "update .github/workflows/release.yml to config your build" required: false + workflow_call: + inputs: + is_priv: + required: true + type: boolean + branch: + required: false + type: string + default: "main" + secrets: + ONEFLOW_PRIV_ORG: + required: true + ONEFLOW_PRIV_GH_TOKEN: + required: true + ONEFLOW_PRIV_OSS_BUCKET: + required: true + OSS_ACCESS_KEY_ID: + required: true + OSS_ACCESS_KEY_SECRET: + required: true concurrency: group: release-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} @@ -31,9 +51,17 @@ jobs: steps: - name: Checkout Oneflow-Inc/oneflow uses: actions/checkout@v2 + if: ${{ !inputs.is_priv }} with: ref: ${{ github.event.pull_request.head.sha }} repository: ${{github.event.pull_request.head.repo.full_name}} + - name: Checkout oneflow + uses: actions/checkout@v2 + if: ${{ inputs.is_priv }} + with: + ref: ${{ inputs.branch }} + repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/oneflow + token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }} - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@add-nightly-date name: find cache id: find-cache @@ -58,11 +86,14 @@ jobs: MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/release/${{ matrix.entry }} WHEELHOUSE_DIR: manylinux_wheelhouse OSS_DIR: branch/${{ github.ref_name }}/${{ matrix.entry }}/${{ github.sha }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SHA: ${{ github.sha }} + ONEFLOW_OSS_BUCKET: oneflow-staging needs: [generate-build-matrix] name: Staging Release timeout-minutes: 180 runs-on: [self-hosted, linux, release] - if: github.repository == 'Oneflow-Inc/oneflow' + if: github.repository == 'Oneflow-Inc/oneflow' || inputs.is_priv strategy: fail-fast: false max-parallel: 5 @@ -76,7 +107,31 @@ jobs: python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple python3 -m pip install -U setuptools wheel --user python3 -m pip install oss2 --user - - uses: actions/checkout@v2 + - name: Checkout Oneflow-Inc/oneflow + uses: actions/checkout@v2 + if: ${{ !inputs.is_priv }} + with: + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{github.event.pull_request.head.repo.full_name}} + - name: Checkout oneflow + uses: actions/checkout@v2 + if: ${{ inputs.is_priv }} + with: + ref: ${{ inputs.branch }} + repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/oneflow + token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }} + - name: Set Private env + if: ${{ inputs.is_priv }} + run: | + GITHUB_SHA=$(git rev-parse HEAD) + echo "OSS_DIR=branch/${{ inputs.branch }}/${{ matrix.entry }}/${GITHUB_SHA}" >> $GITHUB_ENV + echo "GITHUB_REF_NAME=${{ inputs.branch }}" >> $GITHUB_ENV + echo "GITHUB_SHA=${GITHUB_SHA}" >> $GITHUB_ENV + echo "ONEFLOW_OSS_BUCKET=${{ secrets.ONEFLOW_PRIV_OSS_BUCKET }}" >> $GITHUB_ENV + - name: Print env + if: ${{ inputs.is_priv }} + run: | + env - uses: Oneflow-Inc/get-oneflow@add-nightly-date name: Build ${{ matrix.entry }} if: ${{ matrix.entry =='cu118' }} @@ -155,7 +210,7 @@ jobs: uses: ./.github/actions/upload_oss with: src_path: ${{ env.WHEELHOUSE_DIR }} - oss_dst_path: oss://oneflow-staging/${{ env.OSS_DIR }} + oss_dst_path: oss://${{ env.ONEFLOW_OSS_BUCKET }}/${{ env.OSS_DIR }} oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }} oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }} - name: Update pip index @@ -165,13 +220,13 @@ jobs: run: | python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple python3 -m pip install oss2 beautifulsoup4 --user - python3 tools/create_pip_index.py --dir_key ${{ env.OSS_DIR }} -b oneflow-staging \ - --index_key=branch/${{ github.ref_name }}/${{ matrix.entry }}/index.html \ - --index_key=branch/${{ github.ref_name }}/date/${{ needs.generate-build-matrix.outputs.formatted_date }}/${{ matrix.entry }}/index.html \ + python3 tools/create_pip_index.py --dir_key ${{ env.OSS_DIR }} -b ${{ env.ONEFLOW_OSS_BUCKET }} \ + --index_key=branch/${{ env.GITHUB_REF_NAME }}/${{ matrix.entry }}/index.html \ + --index_key=branch/${{ env.GITHUB_REF_NAME }}/date/${{ needs.generate-build-matrix.outputs.formatted_date }}/${{ matrix.entry }}/index.html \ --index_key=${{ env.OSS_DIR }}/index.html \ - --index_key=commit/${{ github.sha }}/${{ matrix.entry }}/index.html + --index_key=commit/${{ env.GITHUB_SHA }}/${{ matrix.entry }}/index.html - name: Update API docs - if: github.ref == 'refs/heads/master' && matrix.entry == 'cpu' + if: github.ref == 'refs/heads/master' && matrix.entry == 'cpu' && !inputs.is_priv env: READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }} run: |