Priv release (#10347)

.github/workflows/priv_release.yml

@@ -0,0 +1,36 @@
+name: Priv Release
+
+on:
+  push:
+    branches:
+      - priv-release
+  schedule:
+    # beijing: 12 pm.
+    # utc: 4 am.
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+    inputs:
+      priv_branch:
+        required: false
+        default: "main"
+
+concurrency:
+  group: priv-release-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  release:
+    name: Release pip
+    permissions:
+      contents: read
+      pull-requests: write
+    uses: Oneflow-Inc/oneflow/.github/workflows/release.yml@master
+    with:
+      is_priv: true
+      branch: ${{ inputs.priv_branch || 'main' }}
+    secrets:
+      ONEFLOW_PRIV_ORG: ${{ secrets.ONEFLOW_PRIV_ORG }}
+      ONEFLOW_PRIV_GH_TOKEN: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }}
+      ONEFLOW_PRIV_OSS_BUCKET: ${{ secrets.ONEFLOW_PRIV_OSS_BUCKET }}
+      OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
+      OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}

.github/workflows/release.yml

@@ -14,6 +14,26 @@ on:
       placeholder:
         description: "update .github/workflows/release.yml to config your build"
         required: false
+  workflow_call:
+    inputs:
+      is_priv:
+        required: true
+        type: boolean
+      branch:
+        required: false
+        type: string
+        default: "main"
+    secrets:
+      ONEFLOW_PRIV_ORG:
+        required: true
+      ONEFLOW_PRIV_GH_TOKEN:
+        required: true
+      ONEFLOW_PRIV_OSS_BUCKET:
+        required: true
+      OSS_ACCESS_KEY_ID:
+        required: true
+      OSS_ACCESS_KEY_SECRET:
+        required: true
 concurrency:
   group: release-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
@@ -31,9 +51,17 @@ jobs:
     steps:
       - name: Checkout Oneflow-Inc/oneflow
         uses: actions/checkout@v2
+        if: ${{ !inputs.is_priv }}
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           repository: ${{github.event.pull_request.head.repo.full_name}}
+      - name: Checkout oneflow
+        uses: actions/checkout@v2
+        if: ${{ inputs.is_priv }}
+        with:
+          ref: ${{ inputs.branch }}
+          repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/oneflow
+          token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }}
       - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@add-nightly-date
         name: find cache
         id: find-cache
@@ -58,11 +86,14 @@ jobs:
       MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/release/${{ matrix.entry }}
       WHEELHOUSE_DIR: manylinux_wheelhouse
       OSS_DIR: branch/${{ github.ref_name }}/${{ matrix.entry }}/${{ github.sha }}
+      GITHUB_REF_NAME: ${{ github.ref_name }}
+      GITHUB_SHA: ${{ github.sha }}
+      ONEFLOW_OSS_BUCKET: oneflow-staging
     needs: [generate-build-matrix]
     name: Staging Release
     timeout-minutes: 180
     runs-on: [self-hosted, linux, release]
-    if: github.repository == 'Oneflow-Inc/oneflow'
+    if: github.repository == 'Oneflow-Inc/oneflow' || inputs.is_priv
     strategy:
       fail-fast: false
       max-parallel: 5
@@ -76,7 +107,31 @@ jobs:
           python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
           python3 -m pip install -U setuptools wheel --user
           python3 -m pip install oss2  --user
-      - uses: actions/checkout@v2
+      - name: Checkout Oneflow-Inc/oneflow
+        uses: actions/checkout@v2
+        if: ${{ !inputs.is_priv }}
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          repository: ${{github.event.pull_request.head.repo.full_name}}
+      - name: Checkout oneflow
+        uses: actions/checkout@v2
+        if: ${{ inputs.is_priv }}
+        with:
+          ref: ${{ inputs.branch }}
+          repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/oneflow
+          token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }}
+      - name: Set Private env
+        if: ${{ inputs.is_priv }}
+        run: |
+          GITHUB_SHA=$(git rev-parse HEAD)
+          echo "OSS_DIR=branch/${{ inputs.branch }}/${{ matrix.entry }}/${GITHUB_SHA}" >> $GITHUB_ENV
+          echo "GITHUB_REF_NAME=${{ inputs.branch }}" >> $GITHUB_ENV
+          echo "GITHUB_SHA=${GITHUB_SHA}" >> $GITHUB_ENV
+          echo "ONEFLOW_OSS_BUCKET=${{ secrets.ONEFLOW_PRIV_OSS_BUCKET }}" >> $GITHUB_ENV
+      - name: Print env
+        if: ${{ inputs.is_priv }}
+        run: |
+          env
       - uses: Oneflow-Inc/get-oneflow@add-nightly-date
         name: Build ${{ matrix.entry }}
         if: ${{ matrix.entry =='cu118' }}
@@ -155,7 +210,7 @@ jobs:
         uses: ./.github/actions/upload_oss
         with:
           src_path: ${{ env.WHEELHOUSE_DIR }}
-          oss_dst_path: oss://oneflow-staging/${{ env.OSS_DIR }}
+          oss_dst_path: oss://${{ env.ONEFLOW_OSS_BUCKET }}/${{ env.OSS_DIR }}
           oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }}
           oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
       - name: Update pip index
@@ -165,13 +220,13 @@ jobs:
         run: |
           python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
           python3 -m pip install oss2 beautifulsoup4 --user
-          python3 tools/create_pip_index.py --dir_key ${{ env.OSS_DIR }} -b oneflow-staging \
-            --index_key=branch/${{ github.ref_name }}/${{ matrix.entry }}/index.html \
-            --index_key=branch/${{ github.ref_name }}/date/${{ needs.generate-build-matrix.outputs.formatted_date }}/${{ matrix.entry }}/index.html \
+          python3 tools/create_pip_index.py --dir_key ${{ env.OSS_DIR }} -b ${{ env.ONEFLOW_OSS_BUCKET }} \
+            --index_key=branch/${{ env.GITHUB_REF_NAME }}/${{ matrix.entry }}/index.html \
+            --index_key=branch/${{ env.GITHUB_REF_NAME }}/date/${{ needs.generate-build-matrix.outputs.formatted_date }}/${{ matrix.entry }}/index.html \
             --index_key=${{ env.OSS_DIR }}/index.html \
-            --index_key=commit/${{ github.sha }}/${{ matrix.entry }}/index.html
+            --index_key=commit/${{ env.GITHUB_SHA }}/${{ matrix.entry }}/index.html
       - name: Update API docs
-        if: github.ref == 'refs/heads/master' && matrix.entry == 'cpu'
+        if: github.ref == 'refs/heads/master' && matrix.entry == 'cpu' && !inputs.is_priv
         env:
           READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }}
         run: |