diff --git a/.github/workflows/compatibility-e2e-v1.yml b/.github/workflows/compatibility-e2e-v1.yml index 7944d822812..19bfc7a105d 100644 --- a/.github/workflows/compatibility-e2e-v1.yml +++ b/.github/workflows/compatibility-e2e-v1.yml @@ -131,7 +131,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Setup Kind - uses: helm/kind-action@v1.9.0 + uses: helm/kind-action@v1.10.0 with: version: ${{ env.KIND_VERSION }} config: ${{ env.KIND_CONFIG_PATH }} diff --git a/.github/workflows/compatibility-e2e-v2.yml b/.github/workflows/compatibility-e2e-v2.yml index 9adca2cb2c5..ceaf445524e 100644 --- a/.github/workflows/compatibility-e2e-v2.yml +++ b/.github/workflows/compatibility-e2e-v2.yml @@ -28,19 +28,19 @@ jobs: include: - module: manager image: manager - image-tag: v2.1.42 + image-tag: v2.1.45 chart-name: manager - module: scheduler image: scheduler - image-tag: v2.1.42 + image-tag: v2.1.45 chart-name: scheduler - module: client image: client - image-tag: v0.1.35 + image-tag: v0.1.51 chart-name: client - module: seed-client image: client - image-tag: v0.1.35 + image-tag: v0.1.51 chart-name: seed-client steps: @@ -117,7 +117,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Setup Kind - uses: helm/kind-action@v1.9.0 + uses: helm/kind-action@v1.10.0 with: version: ${{ env.KIND_VERSION }} config: ${{ env.KIND_CONFIG_PATH }} diff --git a/.github/workflows/e2e-v1-nydus.yml b/.github/workflows/e2e-v1-nydus.yml index 6da9cd60281..aa2c77ca96b 100644 --- a/.github/workflows/e2e-v1-nydus.yml +++ b/.github/workflows/e2e-v1-nydus.yml @@ -77,7 +77,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Setup Kind - uses: helm/kind-action@v1.9.0 + uses: helm/kind-action@v1.10.0 with: version: ${{ env.KIND_VERSION }} config: ${{ env.KIND_CONFIG_PATH }} diff --git a/.github/workflows/e2e-v1.yml b/.github/workflows/e2e-v1.yml index 0d38272b496..4a53fb50adf 100644 --- a/.github/workflows/e2e-v1.yml +++ b/.github/workflows/e2e-v1.yml @@ -150,7 +150,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Setup Kind - uses: helm/kind-action@v1.9.0 + uses: helm/kind-action@v1.10.0 with: version: ${{ env.KIND_VERSION }} config: ${{ env.KIND_CONFIG_PATH }} diff --git a/.github/workflows/e2e-v2-nydus.yml b/.github/workflows/e2e-v2-nydus.yml index 66d0b352bfc..df33eab30d4 100644 --- a/.github/workflows/e2e-v2-nydus.yml +++ b/.github/workflows/e2e-v2-nydus.yml @@ -24,10 +24,22 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Checkout code uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: 0 - name: Setup buildx uses: docker/setup-buildx-action@v3 @@ -130,17 +142,31 @@ jobs: if: always() continue-on-error: true run: | - log_dir="/tmp/nydus-log" - mkdir $log_dir - export ns=nydus-system - for p in `kubectl -n $ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do kubectl -n $ns get pod $p -o yaml >> $log_dir/nydus-pods.log; done - for p in `kubectl -n $ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do kubectl -n $ns describe pod $p >> $log_dir/nydus-pods.log; done - - docker exec kind-control-plane cat /etc/containerd/config.toml >> $log_dir/containerd-config.toml - docker exec kind-control-plane containerd config dump >> $log_dir/containerd-config-dump.toml - - docker exec kind-control-plane journalctl -u containerd >> $log_dir/containerd.log - docker exec kind-control-plane journalctl -u kubelet >> $log_dir/kubelet.log + # Dump nydus logs. + nydus_log_dir="/tmp/nydus" + mkdir -p $nydus_log_dir + export nydus_ns=nydus-system + for p in `kubectl -n $nydus_ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do kubectl -n $nydus_ns get pod $p -o yaml >> $nydus_log_dir/nydus-pods.log; done + for p in `kubectl -n $nydus_ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do kubectl -n $nydus_ns describe pod $p >> $nydus_log_dir/nydus-pods.log; done + for p in `kubectl -n $nydus_ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do kubectl -n $nydus_ns logs pod $p >> $nydus_log_dir/nydus-stdout.log; done + + docker exec kind-control-plane cat /etc/containerd/config.toml >> $nydus_log_dir/containerd-config.toml + docker exec kind-control-plane containerd config dump >> $nydus_log_dir/containerd-config-dump.toml + + docker exec kind-control-plane journalctl -u containerd >> $nydus_log_dir/containerd.log + docker exec kind-control-plane journalctl -u kubelet >> $nydus_log_dir/kubelet.log + + # Dump dragonfly client logs. + dragonfly_log_dir="tmp/dragonfly" + mkdir -p $dragonfly_log_dir + export dragonfly_ns=dragonfly-system + + pod_names=$(kubectl get pods -l 'component in (client, seed-client)' -o custom-columns=NAME:metadata.name --no-headers) + echo $pod_names + for pod_name in $pod_names; do + mkdir -p "/tmp/dragonfly/${pod_name}" + kubectl cp "default/${pod_name}:/var/log/dragonfly" "/tmp/dragonfly/${pod_name}" + done - name: Upload Logs uses: actions/upload-artifact@v4 @@ -148,4 +174,5 @@ jobs: with: name: nydus-e2e-tests-logs path: | - /tmp/nydus-log + /tmp/nydus + /tmp/dragonfly diff --git a/.github/workflows/e2e-v2.yml b/.github/workflows/e2e-v2.yml index 9c564a51a56..05f7ef5a8b9 100644 --- a/.github/workflows/e2e-v2.yml +++ b/.github/workflows/e2e-v2.yml @@ -104,7 +104,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache-new - name: Setup Kind - uses: helm/kind-action@v1.9.0 + uses: helm/kind-action@v1.10.0 with: version: ${{ env.KIND_VERSION }} config: ${{ env.KIND_CONFIG_PATH }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6924dff52fc..6aae137f4bb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,7 +24,7 @@ jobs: cache: false - name: Golangci lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v5 with: version: v1.54 args: --verbose diff --git a/ADOPTERS.md b/ADOPTERS.md index 74f0782a9d4..3cf573bad5b 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -93,3 +93,5 @@ refer to [Setup Dragonfly in Kubernetes](https://d7y.io/docs/getting-started/qui **_[Yipitdata](https://www.yipitdata.com/)_** - Large-scale image and file distribution. **_[Amap](https://mobile.amap.com/)_** - Large-scale image and file distribution. + +**_[iQIYI](https://www.iqiyi.com/)_** - Large-scale image distribution. diff --git a/client-rs b/client-rs index da1682693e0..853b4fcc463 160000 --- a/client-rs +++ b/client-rs @@ -1 +1 @@ -Subproject commit da1682693e0deb3342e3f9606436e9d51ea16290 +Subproject commit 853b4fcc4639328d0d4eaa16aee78842f9818f04 diff --git a/client/daemon/peer/peertask_conductor.go b/client/daemon/peer/peertask_conductor.go index 49009e6cb7f..baff9032fe7 100644 --- a/client/daemon/peer/peertask_conductor.go +++ b/client/daemon/peer/peertask_conductor.go @@ -1086,11 +1086,16 @@ func (pt *peerTaskConductor) downloadPiece(workerID int32, request *DownloadPiec pt.runningPieces.Set(request.piece.PieceNum) pt.runningPiecesLock.Unlock() - defer func() { - pt.runningPiecesLock.Lock() - pt.runningPieces.Clean(request.piece.PieceNum) - pt.runningPiecesLock.Unlock() - }() + var cleanRunningPieceDone bool + cleanRunningPiece := func() { + if cleanRunningPieceDone { + cleanRunningPieceDone = true + pt.runningPiecesLock.Lock() + pt.runningPieces.Clean(request.piece.PieceNum) + pt.runningPiecesLock.Unlock() + } + } + defer cleanRunningPiece() ctx, span := tracer.Start(pt.pieceDownloadCtx, fmt.Sprintf(config.SpanDownloadPiece, request.piece.PieceNum)) span.SetAttributes(config.AttributePiece.Int(int(request.piece.PieceNum))) @@ -1117,6 +1122,9 @@ func (pt *peerTaskConductor) downloadPiece(workerID int32, request *DownloadPiec pt.Infof("switch to back source, skip send failed piece") return result } + + // clean running piece first + cleanRunningPiece() attempt, success := pt.pieceTaskSyncManager.acquire( &commonv1.PieceTaskRequest{ Limit: 1, diff --git a/client/daemon/rpcserver/subscriber.go b/client/daemon/rpcserver/subscriber.go index 4028a9dd050..a9382f7adff 100644 --- a/client/daemon/rpcserver/subscriber.go +++ b/client/daemon/rpcserver/subscriber.go @@ -115,9 +115,13 @@ func searchNextPieceNum(sentMap map[int32]struct{}, cur uint32) (nextPieceNum ui } // sendExistPieces will send as much as possible pieces -func (s *subscriber) sendExistPieces(startNum uint32) (total int32, err error) { +func (s *subscriber) sendExistPieces(startNum uint32) error { s.request.StartNum = startNum - return sendExistPieces(s.sync.Context(), s.SugaredLoggerOnWith, s.getPieces, s.request, s.sync, s.sentMap, true) + total, err := sendExistPieces(s.sync.Context(), s.SugaredLoggerOnWith, s.getPieces, s.request, s.sync, s.sentMap, true) + if total > -1 && s.isUnknownTotalPieces() { + s.totalPieces = total + } + return err } func (s *subscriber) receiveRemainingPieceTaskRequests() { @@ -161,9 +165,23 @@ func (s *subscriber) receiveRemainingPieceTaskRequests() { } } +// totalPieces is -1, 0, n +func (s *subscriber) isKnownTotalPieces() bool { + return s.totalPieces > -1 +} + +func (s *subscriber) isUnknownTotalPieces() bool { + return !s.isKnownTotalPieces() +} + +func (s *subscriber) isAllPiecesSent(nextPieceNum uint32) bool { + return nextPieceNum == uint32(s.totalPieces) +} + func (s *subscriber) sendRemainingPieceTasks() error { // nextPieceNum is the least piece num which did not send to remote peer - // may great then total piece count, check the total piece count when use it + // available values: [0, n], n is total piece count + // when nextPieceNum is n, indicate all pieces done var nextPieceNum uint32 s.Lock() for i := int32(s.skipPieceCount); ; i++ { @@ -173,6 +191,7 @@ func (s *subscriber) sendRemainingPieceTasks() error { } } s.Unlock() + s.Debugf("desired next piece num: %d", nextPieceNum) loop: for { select { @@ -182,51 +201,54 @@ loop: case info := <-s.PieceInfoChannel: s.Infof("receive piece info, num: %d, finished: %v", info.Num, info.Finished) // not desired piece - if s.totalPieces > -1 && uint32(info.Num) < nextPieceNum { + if uint32(info.Num) < nextPieceNum { continue } s.Lock() - total, err := s.sendExistPieces(uint32(info.Num)) + err := s.sendExistPieces(uint32(info.Num)) if err != nil { err = s.saveError(err) s.Unlock() return err } - if total > -1 && s.totalPieces == -1 { - s.totalPieces = total - } - if s.totalPieces > -1 && len(s.sentMap)+int(s.skipPieceCount) == int(s.totalPieces) { - s.Unlock() - break loop - } - if info.Finished { + + nextPieceNum = s.searchNextPieceNum(nextPieceNum) + s.Debugf("update desired next piece num: %d", nextPieceNum) + + if info.Finished && s.isAllPiecesSent(nextPieceNum) { s.Unlock() break loop } - nextPieceNum = s.searchNextPieceNum(nextPieceNum) s.Unlock() case <-s.Success: s.Infof("peer task is success, send remaining pieces") s.Lock() // all pieces already sent // empty piece task will reach sendExistPieces to sync content length and piece count - if s.totalPieces > 0 && nextPieceNum == uint32(s.totalPieces) { + if s.totalPieces > 0 && s.isAllPiecesSent(nextPieceNum) { s.Unlock() break loop } - total, err := s.sendExistPieces(nextPieceNum) + + err := s.sendExistPieces(nextPieceNum) if err != nil { err = s.saveError(err) s.Unlock() return err } - if total > -1 && s.totalPieces == -1 { - s.totalPieces = total + + if s.isUnknownTotalPieces() { + s.Unlock() + msg := "task success, but total pieces is unknown" + s.Errorf(msg) + return dferrors.Newf(commonv1.Code_ClientError, msg) } - if s.totalPieces > -1 && len(s.sentMap)+int(s.skipPieceCount) != int(s.totalPieces) { + + nextPieceNum = s.searchNextPieceNum(nextPieceNum) + if !s.isAllPiecesSent(nextPieceNum) { s.Unlock() - msg := "peer task success, but can not send all pieces" + msg := "task success, but not all pieces are sent out" s.Errorf(msg) return dferrors.Newf(commonv1.Code_ClientError, msg) } diff --git a/deploy/helm-charts b/deploy/helm-charts index 56e0ec0b218..245577b914d 160000 --- a/deploy/helm-charts +++ b/deploy/helm-charts @@ -1 +1 @@ -Subproject commit 56e0ec0b218de1826024a6ceb33b2cfb151e0cbd +Subproject commit 245577b914d4296e776fa5a5cbaeb37421bd1a43 diff --git a/go.mod b/go.mod index 655d31b7653..5f32c5b0595 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module d7y.io/dragonfly/v2 go 1.21 require ( - d7y.io/api/v2 v2.0.110 + d7y.io/api/v2 v2.0.112 github.com/MysteriousPotato/go-lockable v1.0.0 github.com/RichardKnop/machinery v1.10.8 github.com/Showmax/go-fqdn v1.0.0 github.com/VividCortex/mysqlerr v1.0.0 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/appleboy/gin-jwt/v2 v2.9.2 - github.com/aws/aws-sdk-go v1.51.11 + github.com/aws/aws-sdk-go v1.52.2 github.com/bits-and-blooms/bitset v1.13.0 github.com/casbin/casbin/v2 v2.81.0 github.com/casbin/gorm-adapter/v3 v3.5.0 @@ -19,7 +19,7 @@ require ( github.com/deckarep/golang-set/v2 v2.6.0 github.com/distribution/distribution/v3 v3.0.0-20220620080156-3e4f8a0ab147 github.com/docker/distribution v2.8.3+incompatible - github.com/docker/docker v26.0.2+incompatible + github.com/docker/docker v26.1.1+incompatible github.com/docker/go-connections v0.5.0 github.com/docker/go-units v0.4.0 github.com/gaius-qi/ping v1.0.0 @@ -70,12 +70,13 @@ require ( github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 github.com/yl2chen/cidranger v1.0.2 + github.com/zeebo/blake3 v0.2.3 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.47.0 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 - go.opentelemetry.io/otel v1.25.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 + go.opentelemetry.io/otel v1.26.0 go.opentelemetry.io/otel/exporters/jaeger v1.17.0 go.opentelemetry.io/otel/sdk v1.25.0 - go.opentelemetry.io/otel/trace v1.25.0 + go.opentelemetry.io/otel/trace v1.26.0 go.uber.org/atomic v1.11.0 go.uber.org/mock v0.4.0 go.uber.org/zap v1.27.0 @@ -233,7 +234,7 @@ require ( go.mongodb.org/mongo-driver v1.9.1 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel/metric v1.25.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.7.0 // indirect golang.org/x/net v0.24.0 // indirect diff --git a/go.sum b/go.sum index 00d9813f58f..9bd98678571 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -d7y.io/api/v2 v2.0.110 h1:/OXUNf02QKjgqqkkek89Lw4YeD0UxTmmVLb0ouOcqIg= -d7y.io/api/v2 v2.0.110/go.mod h1:L35KjMvZmedXQ6UGHieIddUT1ZZjUAJUqJ9lf52r4fw= +d7y.io/api/v2 v2.0.112 h1:I3HCB3GH1tWbp3pFSPIlx7r+XpPcl8UBcNz0A3Ftink= +d7y.io/api/v2 v2.0.112/go.mod h1:L35KjMvZmedXQ6UGHieIddUT1ZZjUAJUqJ9lf52r4fw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= @@ -156,8 +156,8 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.37.16/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.51.11 h1:El5VypsMIz7sFwAAj/j06JX9UGs4KAbAIEaZ57bNY4s= -github.com/aws/aws-sdk-go v1.51.11/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.52.2 h1:l4g9wBXRBlvCtScvv4iLZCzLCtR7BFJcXOnOGQ20orw= +github.com/aws/aws-sdk-go v1.52.2/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= github.com/aws/aws-sdk-go-v2/service/acmpca v1.6.3/go.mod h1:b8TQvsqtuWslJ4o+eXiHV59P4t20d0EJJ/YVICGJu/o= @@ -400,8 +400,8 @@ github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20200319182547-c7ad2b866182/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v26.0.2+incompatible h1:yGVmKUFGgcxA6PXWAokO0sQL22BrQ67cgVjko8tGdXE= -github.com/docker/docker v26.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v26.1.1+incompatible h1:oI+4kkAgIwwb54b9OC7Xc3hSgu1RlJA/Lln/DF72djQ= +github.com/docker/docker v26.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -948,6 +948,7 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= @@ -1484,6 +1485,12 @@ github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= @@ -1515,22 +1522,22 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.47.0 h1:klI20G/ha94DQjyGuZ8Ajzi3B0C/kVFOESf58tMRq/8= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.47.0/go.mod h1:uVxaSGXSHkn60f5XyeNe4UVg+4eXVxmi0fg1ja42uCQ= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/contrib/propagators/b3 v1.22.0 h1:Okbgv0pWHMQq+mF7H2o1mucJ5PvxKFq2c8cyqoXfeaQ= go.opentelemetry.io/contrib/propagators/b3 v1.22.0/go.mod h1:N3z0ycFRhsVZ+tG/uavMxHvOvFE95QM6gwW1zSqT9dQ= -go.opentelemetry.io/otel v1.25.0 h1:gldB5FfhRl7OJQbUHt/8s0a7cE8fbsPAtdpRaApKy4k= -go.opentelemetry.io/otel v1.25.0/go.mod h1:Wa2ds5NOXEMkCmUou1WA7ZBfLTHWIsp034OVD7AO+Vg= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= -go.opentelemetry.io/otel/metric v1.25.0 h1:LUKbS7ArpFL/I2jJHdJcqMGxkRdxpPHE0VU/D4NuEwA= -go.opentelemetry.io/otel/metric v1.25.0/go.mod h1:rkDLUSd2lC5lq2dFNrX9LGAbINP5B7WBkC78RXCpH5s= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= go.opentelemetry.io/otel/sdk v1.25.0 h1:PDryEJPC8YJZQSyLY5eqLeafHtG+X7FWnf3aXMtxbqo= go.opentelemetry.io/otel/sdk v1.25.0/go.mod h1:oFgzCM2zdsxKzz6zwpTZYLLQsFwc+K0daArPdIhuxkw= -go.opentelemetry.io/otel/trace v1.25.0 h1:tqukZGLwQYRIFtSQM2u2+yfMVTgGVeqRLPUYx1Dq6RM= -go.opentelemetry.io/otel/trace v1.25.0/go.mod h1:hCCs70XM/ljO+BeQkyFnbK28SBIJ/Emuha+ccrCRT7I= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/pkg/digest/digest.go b/pkg/digest/digest.go index 539cf22c496..f377c516651 100644 --- a/pkg/digest/digest.go +++ b/pkg/digest/digest.go @@ -29,9 +29,14 @@ import ( "io" "os" "strings" + + "github.com/zeebo/blake3" ) const ( + // AlgorithmBlake3 is blake3 algorithm name of hash. + AlgorithmBlake3 = "blake3" + // AlgorithmSHA1 is sha1 algorithm name of hash. AlgorithmSHA1 = "sha1" @@ -77,6 +82,8 @@ func HashFile(path string, algorithm string) (string, error) { var h hash.Hash switch algorithm { + case AlgorithmBlake3: + h = blake3.New() case AlgorithmSHA1: h = sha1.New() case AlgorithmSHA256: @@ -109,6 +116,10 @@ func Parse(digest string) (*Digest, error) { encoded := values[1] switch algorithm { + case AlgorithmBlake3: + if len(encoded) != 64 { + return nil, errors.New("invalid encoded") + } case AlgorithmSHA1: if len(encoded) != 40 { return nil, errors.New("invalid encoded") diff --git a/scheduler/service/service_v2.go b/scheduler/service/service_v2.go index 27ce8978963..86803ba0bed 100644 --- a/scheduler/service/service_v2.go +++ b/scheduler/service/service_v2.go @@ -1014,7 +1014,7 @@ func (v *V2) handleDownloadPeerFinishedRequest(ctx context.Context, peerID strin metrics.DownloadPeerCount.WithLabelValues(priority.String(), peer.Task.Type.String(), peer.Task.Tag, peer.Task.Application, peer.Host.Type.Name()).Inc() // TODO to be determined which traffic type to use, temporarily use TrafficType_REMOTE_PEER instead - metrics.DownloadPeerDuration.WithLabelValues(metrics.CalculateSizeLevel(peer.Task.ContentLength.Load()).String()).Observe(float64(peer.Cost.Load())) + metrics.DownloadPeerDuration.WithLabelValues(metrics.CalculateSizeLevel(peer.Task.ContentLength.Load()).String()).Observe(float64(peer.Cost.Load().Milliseconds())) return nil } @@ -1047,7 +1047,7 @@ func (v *V2) handleDownloadPeerBackToSourceFinishedRequest(ctx context.Context, metrics.DownloadPeerCount.WithLabelValues(priority.String(), peer.Task.Type.String(), peer.Task.Tag, peer.Task.Application, peer.Host.Type.Name()).Inc() // TODO to be determined which traffic type to use, temporarily use TrafficType_REMOTE_PEER instead - metrics.DownloadPeerDuration.WithLabelValues(metrics.CalculateSizeLevel(peer.Task.ContentLength.Load()).String()).Observe(float64(peer.Cost.Load())) + metrics.DownloadPeerDuration.WithLabelValues(metrics.CalculateSizeLevel(peer.Task.ContentLength.Load()).String()).Observe(float64(peer.Cost.Load().Milliseconds())) return nil } diff --git a/test/e2e/v2/dfget_test.go b/test/e2e/v2/dfget_test.go index 18604b722e1..65870d7651f 100644 --- a/test/e2e/v2/dfget_test.go +++ b/test/e2e/v2/dfget_test.go @@ -40,7 +40,7 @@ var _ = Describe("Download Using Dfget", func() { fileMetadata := util.FileMetadata{ ID: "1fc5ed9922a3d741063c169ec49c2071a391db5fda8de30eb6a97f60b5038c16", - Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641", + Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", } sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) @@ -230,7 +230,7 @@ var _ = Describe("Download Using Dfget", func() { fileMetadata := util.FileMetadata{ ID: "1ae51fe69c381a4604517f1d00d4315afef070bab3bcb475f11770fc5b194821", - Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641", + Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", } sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) @@ -420,7 +420,7 @@ var _ = Describe("Download Using Dfget", func() { fileMetadata := util.FileMetadata{ ID: "9746acdb4bd8bf2deeb5dd8a3275e51a7fdd4adf8b0dc1d9d26a4565d3ed6592", - Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641", + Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", } sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) @@ -458,7 +458,7 @@ var _ = Describe("Download Using Dfget", func() { fileMetadata := util.FileMetadata{ ID: "9746acdb4bd8bf2deeb5dd8a3275e51a7fdd4adf8b0dc1d9d26a4565d3ed6592", - Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641", + Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", } sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) diff --git a/test/e2e/v2/proxy_test.go b/test/e2e/v2/proxy_test.go index f6980038f39..7306c53c1c6 100644 --- a/test/e2e/v2/proxy_test.go +++ b/test/e2e/v2/proxy_test.go @@ -40,7 +40,7 @@ var _ = Describe("Download Using Proxy", func() { fileMetadata := util.FileMetadata{ ID: "0ef882badaeb3195aed759a203cfb61951d158bf614f90ab0a20504fb7f97992", - Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641", + Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", } sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) @@ -498,7 +498,7 @@ var _ = Describe("Download Using Prefetch Proxy", func() { fileMetadata := util.FileMetadata{ ID: "0fa5f40dfccfcd4c20b2d2dba0bab7205bfced189cc7a013919a6d155d841dab", - Sha256: "66404431f9a0d5c78205e5a3eb041f76767094fc278c0a091d4ffa10f06cf641", + Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", } sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) diff --git a/test/testdata/charts/config-nydus.yaml b/test/testdata/charts/config-nydus.yaml index 96850f3230b..1294cba101a 100644 --- a/test/testdata/charts/config-nydus.yaml +++ b/test/testdata/charts/config-nydus.yaml @@ -18,7 +18,6 @@ dragonfly: "X-Dragonfly-Registry": "https://ghcr.io" ping_url: "http://127.0.0.1:40901/server/ping" - containerRuntime: initContainerImage: ghcr.io/liubin/toml-cli:v0.0.7 containerd: diff --git a/test/testdata/charts/config-v2-nydus.yaml b/test/testdata/charts/config-v2-nydus.yaml index 9226c6e965d..90a8f931754 100644 --- a/test/testdata/charts/config-v2-nydus.yaml +++ b/test/testdata/charts/config-v2-nydus.yaml @@ -18,7 +18,6 @@ dragonfly: "X-Dragonfly-Registry": "https://ghcr.io" ping_url: "http://127.0.0.1:4003/healthy" - containerRuntime: initContainerImage: ghcr.io/liubin/toml-cli:v0.0.7 containerd: diff --git a/test/testdata/containerd/config-v2.toml b/test/testdata/containerd/config-v2.toml new file mode 100644 index 00000000000..980316bdf79 --- /dev/null +++ b/test/testdata/containerd/config-v2.toml @@ -0,0 +1,30 @@ +# explicitly use v2 config format +version = 2 + +[debug] + level = "debug" + +[plugins."io.containerd.grpc.v1.cri".containerd] + # save disk space when using a single snapshotter + discard_unpacked_layers = false + # explicitly use default snapshotter so we can sed it in entrypoint + snapshotter = "overlayfs" + # explicit default here, as we're configuring it below + default_runtime_name = "runc" +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + # set default runtime handler to v2, which has a per-pod shim + runtime_type = "io.containerd.runc.v2" + +# Setup a runtime with the magic name ("test-handler") used for Kubernetes +# runtime class tests ... +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler] + runtime_type = "io.containerd.runc.v2" + +[plugins."io.containerd.grpc.v1.cri"] + # use fixed sandbox image + sandbox_image = "registry.k8s.io/pause:3.5" + # allow hugepages controller to be missing + # see https://github.com/containerd/cri/pull/1501 + tolerate_missing_hugepages_controller = true + # restrict_oom_score_adj needs to be true when running inside UserNS (rootless) + restrict_oom_score_adj = false diff --git a/test/testdata/kind/config-v2.yaml b/test/testdata/kind/config-v2.yaml index 6b2c5f0c6f9..644e910e230 100644 --- a/test/testdata/kind/config-v2.yaml +++ b/test/testdata/kind/config-v2.yaml @@ -9,7 +9,12 @@ nodes: - containerPort: 4001 hostPort: 4001 protocol: TCP + - containerPort: 4003 + hostPort: 4003 + protocol: TCP extraMounts: + - hostPath: ./test/testdata/containerd/config-v2.toml + containerPath: /etc/containerd/config.toml - hostPath: /tmp/artifact containerPath: /tmp/artifact - hostPath: /dev/fuse