From 4c135a02738cc7ead0ff557e1880e9031b3fd22c Mon Sep 17 00:00:00 2001 From: "suhan.zcy" Date: Thu, 2 Jan 2025 11:00:54 +0800 Subject: [PATCH] refactor: refactor the e2e file server Signed-off-by: suhan.zcy --- .github/workflows/compatibility-e2e-v2.yml | 7 +- .github/workflows/e2e-v2.yml | 9 +- test/e2e/v2/concurrency_test.go | 76 +- test/e2e/v2/containerd_test.go | 16 +- test/e2e/v2/dfget_test.go | 259 ++++--- test/e2e/v2/e2e_test.go | 10 + test/e2e/v2/manager/preheat.go | 87 ++- test/e2e/v2/manager/task.go | 99 ++- test/e2e/v2/proxy_test.go | 860 ++++++++++++++------- test/e2e/v2/util/dufs.go | 118 +++ test/e2e/v2/util/file.go | 160 ++++ test/e2e/v2/util/file_server.go | 70 +- test/e2e/v2/util/task.go | 43 ++ test/testdata/k8s/dufs.yaml | 63 ++ 14 files changed, 1363 insertions(+), 514 deletions(-) create mode 100644 test/e2e/v2/util/dufs.go create mode 100644 test/e2e/v2/util/file.go create mode 100644 test/testdata/k8s/dufs.yaml diff --git a/.github/workflows/compatibility-e2e-v2.yml b/.github/workflows/compatibility-e2e-v2.yml index d8b220c8a15..eb12e947dc7 100644 --- a/.github/workflows/compatibility-e2e-v2.yml +++ b/.github/workflows/compatibility-e2e-v2.yml @@ -19,7 +19,7 @@ env: KIND_CONFIG_PATH: test/testdata/kind/config-v2.yaml DRAGONFLY_CHARTS_CONFIG_PATH: test/testdata/charts/config-v2.yaml DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly - DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml + DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/dufs.yaml jobs: compatibility_e2e_tests: @@ -71,7 +71,7 @@ jobs: - name: Get dependencies run: | - go install github.com/onsi/ginkgo/v2/ginkgo@v2.12.0 + go install github.com/onsi/ginkgo/v2/ginkgo@v2.22.1 mkdir -p /tmp/artifact - name: Setup buildx @@ -136,8 +136,9 @@ jobs: - name: Setup dragonfly run: | helm install --wait --timeout 15m --dependency-update --create-namespace --namespace dragonfly-system --set ${{ matrix.chart-name }}.image.tag=${{ matrix.image-tag }} --set ${{ matrix.chart-name }}.image.repository=dragonflyoss/${{ matrix.image }} -f ${{ env.DRAGONFLY_CHARTS_CONFIG_PATH }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }} + mkdir -p /tmp/artifact/dufs && chmod 777 /tmp/artifact/dufs kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }} - kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po dufs-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m - name: Run E2E test run: | diff --git a/.github/workflows/e2e-v2.yml b/.github/workflows/e2e-v2.yml index 5385f03026d..440d8dd035d 100644 --- a/.github/workflows/e2e-v2.yml +++ b/.github/workflows/e2e-v2.yml @@ -10,7 +10,7 @@ on: schedule: - cron: '0 4 * * *' -permissions: +permissions: contents: read env: @@ -18,7 +18,7 @@ env: CONTAINERD_VERSION: v1.5.2 KIND_CONFIG_PATH: test/testdata/kind/config-v2.yaml DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly - DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml + DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/dufs.yaml jobs: e2e_tests: @@ -58,7 +58,7 @@ jobs: - name: Get dependencies run: | - go install github.com/onsi/ginkgo/v2/ginkgo@v2.12.0 + go install github.com/onsi/ginkgo/v2/ginkgo@v2.22.1 mkdir -p /tmp/artifact - name: Setup buildx @@ -123,8 +123,9 @@ jobs: - name: Setup dragonfly run: | helm install --wait --timeout 15m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }} + mkdir -p /tmp/artifact/dufs && chmod 777 /tmp/artifact/dufs kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }} - kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl wait po dufs-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m - name: Run E2E test run: | diff --git a/test/e2e/v2/concurrency_test.go b/test/e2e/v2/concurrency_test.go index 4a5cbd6e2c1..a82991071c3 100644 --- a/test/e2e/v2/concurrency_test.go +++ b/test/e2e/v2/concurrency_test.go @@ -27,23 +27,34 @@ import ( var _ = Describe("Download Concurrency", func() { Context("ab", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("concurrent 100 should be ok", Label("concurrent", "100"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 100 -n 200 -X 127.0.0.1:4001 %s", util.GetFileURL("/bin/unshare"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 100 -n 200 -X 127.0.0.1:4001 %s", testFile.GetDownloadURL())).CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "14b31801ea6990788057b965fbc51e44bf73800462915fdfa0fda8182acca4d6", - Sha256: "fc44bbbba20490450c73530db3d1b935f893f38d7d8084ca132952a765ff5ff6", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { @@ -52,9 +63,9 @@ var _ = Describe("Download Concurrency", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) It("concurrent 200 should be ok", Label("concurrent", "200"), func() { @@ -62,18 +73,13 @@ var _ = Describe("Download Concurrency", func() { fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 200 -n 400 -X 127.0.0.1:4001 %s", util.GetFileURL("/bin/loginctl"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 200 -n 400 -X 127.0.0.1:4001 %s", testFile.GetDownloadURL())).CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "958e177b56be708c9d7ec193ae8cef399b39faff8234af33efa4cbe097d1fc5f", - Sha256: "dc102987a36be20846821ac74648534863ff0fe8897d4250273a6ffc80481d91", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { @@ -82,9 +88,9 @@ var _ = Describe("Download Concurrency", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) It("concurrent 500 should be ok", Label("concurrent", "500"), func() { @@ -92,18 +98,13 @@ var _ = Describe("Download Concurrency", func() { fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 500 -n 1000 -X 127.0.0.1:4001 %s", util.GetFileURL("/bin/realpath"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 500 -n 1000 -X 127.0.0.1:4001 %s", testFile.GetDownloadURL())).CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "dd573cf9c3e1a79402b8423abcd1ba987c1b1ee9c49069d139d71106a260b055", - Sha256: "54e54b7ff54ef70d4db2adcd24a27e3b9af3cd99fc0213983bac1e8035429be6", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { @@ -112,9 +113,9 @@ var _ = Describe("Download Concurrency", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) It("concurrent 1000 should be ok", Label("concurrent", "1000"), func() { @@ -122,18 +123,13 @@ var _ = Describe("Download Concurrency", func() { fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 1000 -n 2000 -X 127.0.0.1:4001 %s", util.GetFileURL("/bin/lnstat"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 1000 -n 2000 -X 127.0.0.1:4001 %s", testFile.GetDownloadURL())).CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "f1957adc26ec326800ced850d72e583a03be0999ba80d9aa2e3ba57ef4ddaf17", - Sha256: "87c09b7c338f258809ca2d436bbe06ac94a3166b3f3e1125a86f35d9a9aa1d2f", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { @@ -142,9 +138,9 @@ var _ = Describe("Download Concurrency", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) }) diff --git a/test/e2e/v2/containerd_test.go b/test/e2e/v2/containerd_test.go index 6245a4c2444..20c467edea3 100644 --- a/test/e2e/v2/containerd_test.go +++ b/test/e2e/v2/containerd_test.go @@ -29,7 +29,7 @@ import ( var _ = Describe("Containerd with CRI support", func() { Context("dragonflyoss/manager:v2.0.0 image", func() { It("pull should be ok", Label("containerd", "pull"), func() { - out, err := util.CriCtlCommand("pull", "dragonflyoss/manager:v2.1.0").CombinedOutput() + out, err := util.CriCtlCommand("pull", "index.docker.io/dragonflyoss/manager:v2.1.0").CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) @@ -84,7 +84,7 @@ var _ = Describe("Containerd with CRI support", func() { }) It("rmi should be ok", Label("containerd", "rmi"), func() { - out, err := util.CriCtlCommand("rmi", "dragonflyoss/manager:v2.1.0").CombinedOutput() + out, err := util.CriCtlCommand("rmi", "index.docker.io/dragonflyoss/manager:v2.1.0").CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) }) @@ -92,7 +92,7 @@ var _ = Describe("Containerd with CRI support", func() { Context("dragonflyoss/scheduler:v2.0.0 image", func() { It("pull should be ok", Label("containerd", "pull"), func() { - out, err := util.CriCtlCommand("pull", "dragonflyoss/scheduler:v2.0.0").CombinedOutput() + out, err := util.CriCtlCommand("pull", "index.docker.io/dragonflyoss/scheduler:v2.0.0").CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) @@ -140,7 +140,7 @@ var _ = Describe("Containerd with CRI support", func() { }) It("rmi should be ok", Label("containerd", "rmi"), func() { - out, err := util.CriCtlCommand("rmi", "dragonflyoss/scheduler:v2.0.0").CombinedOutput() + out, err := util.CriCtlCommand("rmi", "index.docker.io/dragonflyoss/scheduler:v2.0.0").CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) }) @@ -148,7 +148,7 @@ var _ = Describe("Containerd with CRI support", func() { Context("dragonflyoss/client:v0.1.30 image", func() { It("pull should be ok", Label("containerd", "pull"), func() { - out, err := util.CriCtlCommand("pull", "dragonflyoss/client:v0.1.30").CombinedOutput() + out, err := util.CriCtlCommand("pull", "index.docker.io/dragonflyoss/client:v0.1.30").CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) @@ -204,7 +204,7 @@ var _ = Describe("Containerd with CRI support", func() { }) It("rmi should be ok", Label("containerd", "rmi"), func() { - out, err := util.CriCtlCommand("rmi", "dragonflyoss/client:v0.1.30").CombinedOutput() + out, err := util.CriCtlCommand("rmi", "index.docker.io/dragonflyoss/client:v0.1.30").CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) }) @@ -212,7 +212,7 @@ var _ = Describe("Containerd with CRI support", func() { Context("dragonflyoss/dfinit:v0.1.30 image", func() { It("pull should be ok", Label("containerd", "pull"), func() { - out, err := util.CriCtlCommand("pull", "dragonflyoss/dfinit:v0.1.30").CombinedOutput() + out, err := util.CriCtlCommand("pull", "index.docker.io/dragonflyoss/dfinit:v0.1.30").CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) @@ -256,7 +256,7 @@ var _ = Describe("Containerd with CRI support", func() { }) It("rmi should be ok", Label("containerd", "rmi"), func() { - out, err := util.CriCtlCommand("rmi", "dragonflyoss/dfinit:v0.1.30").CombinedOutput() + out, err := util.CriCtlCommand("rmi", "index.docker.io/dragonflyoss/dfinit:v0.1.30").CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/v2/dfget_test.go b/test/e2e/v2/dfget_test.go index c6241d1a013..23ab885fe97 100644 --- a/test/e2e/v2/dfget_test.go +++ b/test/e2e/v2/dfget_test.go @@ -27,28 +27,39 @@ import ( ) var _ = Describe("Download Using Dfget", func() { - Context("/etc/containerd/config.toml file", func() { + Context("1MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("dfget", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", util.GetFileURL("/etc/containerd/config.toml"), util.GetOutputPath("config.toml-1"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(string(out), err) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "f82afbc76aa4d6472a982a8b8383a98d22c18c9ab06d24e989d9d4d30a49dfee", - Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("config.toml-1")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -58,34 +69,45 @@ var _ = Describe("Download Using Dfget", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file", func() { + Context("10MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize10MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("dfget", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-1"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(string(out), err) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "144eb7b6f67b8423c9999b9b864a35562aeff3e7d153eed2bb7f917977796bdf", - Sha256: "327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-1")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -95,34 +117,45 @@ var _ = Describe("Download Using Dfget", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/x86_64 file", func() { + Context("100MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize100MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("dfget", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", util.GetFileURL("/bin/x86_64"), util.GetOutputPath("x86_64-1"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(string(out), err) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "7327d3aa45f16e60396d687d8c81a9d5ec9c6ca662ad9c6dad77f830c499d5e3", - Sha256: "a1cbf1bf2d66757121677fd7fefafacd4f843a2cb44a451131002803bae56a65", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("x86_64-1")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -132,34 +165,45 @@ var _ = Describe("Download Using Dfget", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/zless file", func() { + Context("500MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize500MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("dfget", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", util.GetFileURL("/bin/zless"), util.GetOutputPath("zless-1"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(string(out), err) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "92709a5ce8a45db2563081b843403eec5d72f8a78fe2be85ce23e0edf8b2d207", - Sha256: "b0cfe211f851049a78f5812cf5b1d7cb4f3fbb101c02c1865c940d5797f4b73b", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("zless-1")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -169,34 +213,45 @@ var _ = Describe("Download Using Dfget", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/bash file", func() { + Context("1GiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1GiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("dfget", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", util.GetFileURL("/bin/bash"), util.GetOutputPath("bash-1"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(string(out), err) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "56ac4873dda3db55ed78c6b26b24d51343803882714a54802bfddbb1097eb06d", - Sha256: "c37f93c73cf2f303f874c094f6f76e47b2421a3da9f0e7e0b98bea8a3d685322", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("bash-1")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -206,34 +261,45 @@ var _ = Describe("Download Using Dfget", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/etc/containerd/config.toml file and set application to d7y", func() { + Context("1MiB file and set application to d7y", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("dfget", "download", "application d7y"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --application d7y --output %s", util.GetFileURL("/etc/containerd/config.toml"), util.GetOutputPath("config.toml-3"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --application d7y --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(string(out), err) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "fbf82ee8c8fb2f234c46947e95b4b51fc9a01b9306e52c6b717f5323b6e7c5e4", - Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDApplication("d7y"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("config.toml-3")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -243,34 +309,45 @@ var _ = Describe("Download Using Dfget", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDApplication("d7y"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/etc/containerd/config.toml file and set tag to d7y", func() { + Context("1MiB file and set tag to d7y", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("dfget", "download", "tag d7y"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --tag d7y --output %s", util.GetFileURL("/etc/containerd/config.toml"), util.GetOutputPath("config.toml-4"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --tag d7y --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(string(out), err) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "fbf82ee8c8fb2f234c46947e95b4b51fc9a01b9306e52c6b717f5323b6e7c5e4", - Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("d7y"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("config.toml-4")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -280,9 +357,9 @@ var _ = Describe("Download Using Dfget", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("d7y"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) }) diff --git a/test/e2e/v2/e2e_test.go b/test/e2e/v2/e2e_test.go index 0cad14fd471..f7ff2b1ab53 100644 --- a/test/e2e/v2/e2e_test.go +++ b/test/e2e/v2/e2e_test.go @@ -80,9 +80,19 @@ var _ = AfterSuite(func() { fmt.Printf("------------------------------ Get %s-%d Artifact Finished ------------------------------\n", server.Name, i) } } + + // Clean up file server. + if err := util.GetFileServer().Purge(); err != nil { + fmt.Printf("failed to purge the e2e file server: %v\n", err) + } + }) var _ = BeforeSuite(func() { + fs, err := util.NewFileServer() + Expect(err).NotTo(HaveOccurred()) + Expect(fs).NotTo(BeNil()) + rawGitCommit, err := util.GitCommand("rev-parse", "--short", "HEAD").CombinedOutput() Expect(err).NotTo(HaveOccurred()) gitCommit := strings.Fields(string(rawGitCommit))[0] diff --git a/test/e2e/v2/manager/preheat.go b/test/e2e/v2/manager/preheat.go index dd4cc0fed85..41ee660ff75 100644 --- a/test/e2e/v2/manager/preheat.go +++ b/test/e2e/v2/manager/preheat.go @@ -31,7 +31,23 @@ import ( ) var _ = Describe("Preheat with Manager", func() { - Context("/bin/md5sum file", func() { + Context("1MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("preheat files should be ok", Label("preheat", "file"), func() { managerPod, err := util.ManagerExec(0) fmt.Println(err) @@ -41,7 +57,7 @@ var _ = Describe("Preheat with Manager", func() { Type: internaljob.PreheatJob, Args: types.PreheatArgs{ Type: "file", - URL: util.GetFileURL("/bin/md5sum"), + URL: testFile.GetDownloadURL(), }, }) Expect(err).NotTo(HaveOccurred()) @@ -60,11 +76,6 @@ var _ = Describe("Preheat with Manager", func() { done := waitForDone(job, managerPod) Expect(done).Should(BeTrue()) - fileMetadata := util.FileMetadata{ - ID: "b0a5cfd4ccf5310803675f742dedc435a64e9a5f539f48fedbef6c30aac18b7c", - Sha256: "80f1d8cd843a98b23b30e90e7e43a14e05935351f354d678bc465f7be66ef3dd", - } - seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedClientPods[i], err = util.SeedClientExec(i) @@ -72,13 +83,29 @@ var _ = Describe("Preheat with Manager", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/toe file", func() { + Context("10MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize10MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("preheat files should be ok", Label("preheat", "file"), func() { managerPod, err := util.ManagerExec(0) fmt.Println(err) @@ -88,7 +115,7 @@ var _ = Describe("Preheat with Manager", func() { Type: internaljob.PreheatJob, Args: types.PreheatArgs{ Type: "file", - URL: util.GetFileURL("/bin/toe"), + URL: testFile.GetDownloadURL(), }, }) Expect(err).NotTo(HaveOccurred()) @@ -107,11 +134,6 @@ var _ = Describe("Preheat with Manager", func() { done := waitForDone(job, managerPod) Expect(done).Should(BeTrue()) - fileMetadata := util.FileMetadata{ - ID: "802e3df5384438deaed066ca445489f6e314ebb6a2d4728d020e75a08d281942", - Sha256: "4c7f0f298ab3350859f90664d706b8ccaa95072f1f1f3dd74f559642e5483cd5", - } - seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedClientPods[i], err = util.SeedClientExec(i) @@ -119,13 +141,29 @@ var _ = Describe("Preheat with Manager", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/jq file", func() { + Context("100MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize100MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("preheat files should be ok", Label("preheat", "file"), func() { managerPod, err := util.ManagerExec(0) fmt.Println(err) @@ -135,7 +173,7 @@ var _ = Describe("Preheat with Manager", func() { Type: internaljob.PreheatJob, Args: types.PreheatArgs{ Type: "file", - URL: util.GetFileURL("/bin/jq"), + URL: testFile.GetDownloadURL(), }, }) Expect(err).NotTo(HaveOccurred()) @@ -154,11 +192,6 @@ var _ = Describe("Preheat with Manager", func() { done := waitForDone(job, managerPod) Expect(done).Should(BeTrue()) - fileMetadata := util.FileMetadata{ - ID: "4f1de4716ec6d1ca56daf1f5dd2520a8f6a826d90474f596cdf99a5c88fef982", - Sha256: "5a963cbdd08df27651e9c9d006567267ebb3c80f7b8fc0f218ade5771df2998b", - } - seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedClientPods[i], err = util.SeedClientExec(i) @@ -166,9 +199,9 @@ var _ = Describe("Preheat with Manager", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) diff --git a/test/e2e/v2/manager/task.go b/test/e2e/v2/manager/task.go index 035d32b4b2b..b029c5a28fc 100644 --- a/test/e2e/v2/manager/task.go +++ b/test/e2e/v2/manager/task.go @@ -31,7 +31,23 @@ import ( ) var _ = Describe("GetTask and DeleteTask with Manager", func() { - Context("/bin/cat file", Label("getTask", "deleteTask", "file"), func() { + Context("1MiB file", Label("getTask", "deleteTask", "file"), func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("getTask and deleteTask should be ok", func() { managerPod, err := util.ManagerExec(0) fmt.Println(err) @@ -41,7 +57,7 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { Type: internaljob.PreheatJob, Args: types.PreheatArgs{ Type: "file", - URL: util.GetFileURL("/bin/cat"), + URL: testFile.GetDownloadURL(), }, SchedulerClusterIDs: []uint{1}, }) @@ -61,11 +77,6 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { done := waitForDone(job, managerPod) Expect(done).Should(BeTrue()) - fileMetadata := util.FileMetadata{ - ID: "5588094e5a40f0d324052e044884a0910de8467428deebccb8c6fb560b131c28", - Sha256: "df954abca766aceddd79dd20429e4f222019018667446626d3a641d3c47c50fc", - } - seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedClientPods[i], err = util.SeedClientExec(i) @@ -73,14 +84,14 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) req, err = structure.StructToMap(types.CreateGetTaskJobRequest{ Type: internaljob.GetTaskJob, Args: types.GetTaskArgs{ - TaskID: fileMetadata.ID, + TaskID: testFile.GetTaskID(), }, }) Expect(err).NotTo(HaveOccurred()) @@ -102,7 +113,7 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { req, err = structure.StructToMap(types.CreateDeleteTaskJobRequest{ Type: internaljob.DeleteTaskJob, Args: types.DeleteTaskArgs{ - TaskID: fileMetadata.ID, + TaskID: testFile.GetTaskID(), }, }) Expect(err).NotTo(HaveOccurred()) @@ -121,12 +132,28 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { Expect(done).Should(BeTrue()) Expect(validateTaskResponse(job)).To(BeTrue()) - exist := util.CheckFilesExist(seedClientPods, fileMetadata.ID) + exist := util.CheckFilesExist(seedClientPods, testFile.GetTaskID()) Expect(exist).Should(BeFalse()) }) }) - Context("/bin/pwd file", Label("getTask", "deleteTask", "file"), func() { + Context("10MiB file", Label("getTask", "deleteTask", "file"), func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize10MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("getTask and deleteTask should be ok", func() { managerPod, err := util.ManagerExec(0) fmt.Println(err) @@ -136,7 +163,7 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { Type: internaljob.PreheatJob, Args: types.PreheatArgs{ Type: "file", - URL: util.GetFileURL("/bin/pwd"), + URL: testFile.GetDownloadURL(), }, SchedulerClusterIDs: []uint{1}, }) @@ -156,11 +183,6 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { done := waitForDone(job, managerPod) Expect(done).Should(BeTrue()) - fileMetadata := util.FileMetadata{ - ID: "feab7bc71cfa2618a2fe4cf593b562d59b4e8cb87e8ec249a1b951f44c78f2b6", - Sha256: "5286873505a9671e077f346cdfb89d5a6c99985fe3f11a972f30fedf9029bae0", - } - seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedClientPods[i], err = util.SeedClientExec(i) @@ -168,14 +190,14 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) req, err = structure.StructToMap(types.CreateGetTaskJobRequest{ Type: internaljob.GetTaskJob, Args: types.GetTaskArgs{ - TaskID: fileMetadata.ID, + TaskID: testFile.GetTaskID(), }, }) Expect(err).NotTo(HaveOccurred()) @@ -197,7 +219,7 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { req, err = structure.StructToMap(types.CreateDeleteTaskJobRequest{ Type: internaljob.DeleteTaskJob, Args: types.DeleteTaskArgs{ - TaskID: fileMetadata.ID, + TaskID: testFile.GetTaskID(), }, }) Expect(err).NotTo(HaveOccurred()) @@ -216,22 +238,33 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { Expect(done).Should(BeTrue()) Expect(validateTaskResponse(job)).To(BeTrue()) - exist := util.CheckFilesExist(seedClientPods, fileMetadata.ID) + exist := util.CheckFilesExist(seedClientPods, testFile.GetTaskID()) Expect(exist).Should(BeFalse()) }) }) - Context("/bin/time file", Label("getTask", "deleteTask", "file"), func() { + Context("100MiB file", Label("getTask", "deleteTask", "file"), func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize100MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("getTask and deleteTask should be failed", func() { managerPod, err := util.ManagerExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "d7704dffb62c1925c56c2ad4e04ba16daf266432eed43a59336e8bd6e71fab92", - Sha256: "7301b9b4c51a8f4d26c1af0da250f03a49ec8a8141033123e79196ad18f6c81b", - } - seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedClientPods[i], err = util.SeedClientExec(i) @@ -239,13 +272,13 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { Expect(err).NotTo(HaveOccurred()) } - exist := util.CheckFilesExist(seedClientPods, fileMetadata.ID) + exist := util.CheckFilesExist(seedClientPods, testFile.GetTaskID()) Expect(exist).Should(BeFalse()) req, err := structure.StructToMap(types.CreateGetTaskJobRequest{ Type: internaljob.GetTaskJob, Args: types.GetTaskArgs{ - TaskID: fileMetadata.ID, + TaskID: testFile.GetTaskID(), }, }) Expect(err).NotTo(HaveOccurred()) @@ -267,7 +300,7 @@ var _ = Describe("GetTask and DeleteTask with Manager", func() { req, err = structure.StructToMap(types.CreateDeleteTaskJobRequest{ Type: internaljob.DeleteTaskJob, Args: types.DeleteTaskArgs{ - TaskID: fileMetadata.ID, + TaskID: testFile.GetTaskID(), }, }) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/v2/proxy_test.go b/test/e2e/v2/proxy_test.go index 482402934a3..ae6a1fe2fbb 100644 --- a/test/e2e/v2/proxy_test.go +++ b/test/e2e/v2/proxy_test.go @@ -27,29 +27,40 @@ import ( ) var _ = Describe("Download Using Proxy", func() { - Context("/etc/containerd/config.toml file", func() { + Context("1MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", util.GetFileURL("/etc/containerd/config.toml"), util.GetOutputPath("config-proxy.coml"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "2203d73a8d0162a86745eb25095a655e2ab866c56c3ebcc3f0bf7d6a4e329317", - Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("config-proxy.coml")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -59,35 +70,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file", func() { + Context("10MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize10MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "6b94aa169d11ba67c1852f14c4df39e5ebf84d1c0d7a6f6ad0ba2f3547883f4f", - Sha256: "327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -97,35 +119,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/x86_64 file", func() { + Context("100MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize100MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", util.GetFileURL("/bin/x86_64"), util.GetOutputPath("x86_64-proxy"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "d6636c250217d958ef2377ca9278811cc3c538f3d29a65729ec7e73bbabf5bb6", - Sha256: "a1cbf1bf2d66757121677fd7fefafacd4f843a2cb44a451131002803bae56a65", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("x86_64-proxy")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -135,35 +168,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/zless file", func() { + Context("500MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize500MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", util.GetFileURL("/bin/zless"), util.GetOutputPath("zless-proxy"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "9c9754927a9b3be5b6a0524c3c6f930a292370d0c29f0617a4b54e94622b330b", - Sha256: "b0cfe211f851049a78f5812cf5b1d7cb4f3fbb101c02c1865c940d5797f4b73b", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("zless-proxy")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -173,35 +217,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/bash file", func() { + Context("1GiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1GiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", util.GetFileURL("/bin/bash"), util.GetOutputPath("bash-proxy"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "91623b36b06c8fe3c87af420b92df060884a1fda1b11951349a0f1d02f9cc2e3", - Sha256: "c37f93c73cf2f303f874c094f6f76e47b2421a3da9f0e7e0b98bea8a3d685322", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("bash-proxy")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -211,35 +266,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=-100", func() { + Context("1MiB file and set range header bytes=-100", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download", "range: bytes=-100"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r -100 -H 'X-Dragonfly-Tag: proxy-bytes-100' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy-bytes-100"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r -100 -H 'X-Dragonfly-Tag: proxy-bytes-100' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "5a7a62911a30fbcdf602b3b37ce3ecb56f4c47ab65283219c5d2342223ee53cb", - Sha256: "cd00e292c5970d3c5e2f0ffa5171e555bc46bfc4faddfb4a418b6840b86e79a3", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100"))) Expect(err).NotTo(HaveOccurred()) - Expect("81d9f55e659a1b764e99dec6915ed47aa66955d30c47ffd4d427b4194a31684d").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy-bytes-100")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("-100", testFile.GetInfo().Size())).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -249,35 +315,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100"))) Expect(err).NotTo(HaveOccurred()) - Expect("81d9f55e659a1b764e99dec6915ed47aa66955d30c47ffd4d427b4194a31684d").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=0-100", func() { + Context("1MiB file and set range header bytes=0-100", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download", "range: bytes=0-100"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 0-100 -H 'X-Dragonfly-Tag: proxy-bytes-0-100' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy-bytes-0-100"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 0-100 -H 'X-Dragonfly-Tag: proxy-bytes-0-100' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "41d7764fb682932aac4e1591ff01739493539e6af9072773624b3e20fe5465f4", - Sha256: "7bbfcb694f6acc69483751d3b48d5bdbdcb284f9e04c7b7caa04b5977cffddc8", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-0-100"))) Expect(err).NotTo(HaveOccurred()) - Expect("0dd6bcd036002bbd79cd8dfebcd9813ec4231562c0b15b02cbc7c70a062717f9").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy-bytes-0-100")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("0-100", testFile.GetInfo().Size())).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -287,35 +364,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-0-100"))) Expect(err).NotTo(HaveOccurred()) - Expect("0dd6bcd036002bbd79cd8dfebcd9813ec4231562c0b15b02cbc7c70a062717f9").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=100-", func() { + Context("1MiB file and set range header bytes=100-", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download", "range: bytes=100-"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100- -H 'X-Dragonfly-Tag: proxy-bytes-100-' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy-bytes-100-"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100- -H 'X-Dragonfly-Tag: proxy-bytes-100-' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "c29e8ee77bf3a0f177269fd1a216a4c404a4fb3778a483c41c8735aadffcbe66", - Sha256: "ba9a10ceceb80562fc124dc9bc94ea2a38e3a71e3e746e2140e6381ac791cdeb", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100-"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy-bytes-100-")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("100-", testFile.GetInfo().Size())).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -325,35 +413,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100-"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=100-46026751", func() { - It("download should be ok", Label("proxy", "download", "range: bytes=100-46026751"), func() { + Context("1MiB file and set range header bytes=100-500", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("download should be ok", Label("proxy", "download", "range: bytes=100-500"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-46026751 -H 'X-Dragonfly-Tag: proxy-bytes-100-46026751' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy-bytes-100-46026751"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-500 -H 'X-Dragonfly-Tag: proxy-bytes-100-500' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "29fe25686d520e3aedd1743fc95dc008e6102e917b33d17b75b2db204718f132", - Sha256: "ba9a10ceceb80562fc124dc9bc94ea2a38e3a71e3e746e2140e6381ac791cdeb", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100-500"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy-bytes-100-46026751")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("100-500", testFile.GetInfo().Size())).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -363,35 +462,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100-500"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=0-46026751", func() { - It("download should be ok", Label("proxy", "download", "range: bytes=0-46026751"), func() { + Context("1MiB file and set range header bytes=0-1048575", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("download should be ok", Label("proxy", "download", "range: bytes=0-1048575"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 0-46026751 -H 'X-Dragonfly-Tag: proxy-bytes-0-46026751' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy-bytes-0-46026751"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 0-1048575 -H 'X-Dragonfly-Tag: proxy-bytes-0-1048575' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "09f0107339e32f9405b15fb093b25db9652104d96d3a8797afffa47f5c4c988b", - Sha256: "327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-0-1048575"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy-bytes-0-46026751")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("0-1048575", testFile.GetInfo().Size())).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -401,35 +511,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-0-1048575"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=100-10240", func() { + Context("1MiB file and set range header bytes=100-10240", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download", "range: bytes=100-10240"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-10240 -H 'X-Dragonfly-Tag: proxy-bytes-100-10240' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy-bytes-100-10240"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-10240 -H 'X-Dragonfly-Tag: proxy-bytes-100-10240' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "7a916912f2182f749d2779ad9c557947d7ee2a2b213705038d2441bf5089f5d0", - Sha256: "8732360b941ad09a5e0e5d5f9891118bc068f6d0d5a56e3c6d483e4600fbc43f", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100-10240"))) Expect(err).NotTo(HaveOccurred()) - Expect("0dd6bcd036002bbd79cd8dfebcd9813ec4231562c0b15b02cbc7c70a062717f9").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy-bytes-100-10240")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("100-10240", testFile.GetInfo().Size())).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -439,35 +560,46 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100-10240"))) Expect(err).NotTo(HaveOccurred()) - Expect("0dd6bcd036002bbd79cd8dfebcd9813ec4231562c0b15b02cbc7c70a062717f9").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) Context("/bin/kubectl file and set range header bytes=100-1024", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("proxy", "download", "range: bytes=100-1024"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-1024 -H 'X-Dragonfly-Tag: proxy-bytes-100-1024' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-proxy-bytes-100-1024"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-1024 -H 'X-Dragonfly-Tag: proxy-bytes-100-1024' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "7a916912f2182f749d2779ad9c557947d7ee2a2b213705038d2441bf5089f5d0", - Sha256: "d6d17dca18b8de59e38da525dc24c47b74fec1a790a9f64afdd6538f4f8fa90e", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100-1024"))) Expect(err).NotTo(HaveOccurred()) - Expect("0dd6bcd036002bbd79cd8dfebcd9813ec4231562c0b15b02cbc7c70a062717f9").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, util.GetOutputPath("kubectl-proxy-bytes-100-1024")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("100-1024", testFile.GetInfo().Size())).To(Equal(sha256sum)) time.Sleep(1 * time.Second) seedClientPods := make([]*util.PodExec, 3) @@ -477,330 +609,462 @@ var _ = Describe("Download Using Proxy", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID(util.WithTaskIDTag("proxy-bytes-100-1024"))) Expect(err).NotTo(HaveOccurred()) - Expect("0dd6bcd036002bbd79cd8dfebcd9813ec4231562c0b15b02cbc7c70a062717f9").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) }) var _ = Describe("Download Using Prefetch Proxy", func() { - Context("/etc/containerd/config.toml file", func() { + Context("1MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", util.GetFileURL("/etc/containerd/config.toml"), util.GetOutputPath("config-prefetch-proxy.coml"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "09d0b6f7a9d6825d4d703b85ec706a87334ba437310301e4a0110af8cc562648", - Sha256: "6288d2a89e2a9611191c25a45de20e94d8d058c75f274a39970d41f60f367e6f", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("config-prefetch-proxy.coml")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file", func() { + Context("10MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize10MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-prefetch-proxy"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "0812fe2767598f19d2430be8153522d916f6e6c3179bd8914941eb2c0e8568e5", - Sha256: "327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("kubectl-prefetch-proxy")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/x86_64 file", func() { + Context("100MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize100MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", util.GetFileURL("/bin/x86_64"), util.GetOutputPath("x86_64-prefetch-proxy"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "d93cf0cd379fbd59194c3038837a551a1151a3aedd1c931fc7324ea0b0edc289", - Sha256: "a1cbf1bf2d66757121677fd7fefafacd4f843a2cb44a451131002803bae56a65", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("x86_64-prefetch-proxy")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/zless file", func() { + Context("500MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize500MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", util.GetFileURL("/bin/zless"), util.GetOutputPath("zless-prefetch-proxy"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "ba68b1b385b2441fef06fb79a64b79367fabf4c31e2d7f21cd37b202d2e02e44", - Sha256: "b0cfe211f851049a78f5812cf5b1d7cb4f3fbb101c02c1865c940d5797f4b73b", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("zless-prefetch-proxy")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/bash file", func() { + Context("1GiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1GiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", util.GetFileURL("/bin/bash"), util.GetOutputPath("bash-prefetch-proxy"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -H 'X-Dragonfly-Tag: prefetch-proxy' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "661e488117503f6c46192e86f587a5b615129cadb6f25b51b9683257a1d70b55", - Sha256: "c37f93c73cf2f303f874c094f6f76e47b2421a3da9f0e7e0b98bea8a3d685322", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy"))) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) - sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("bash-prefetch-proxy")) + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=-100", func() { + Context("1MiB file and set range header bytes=-100", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download", "range: bytes=-100"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r -100 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-prefetch-proxy-bytes-100"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r -100 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "43a914f8b0f3ab0be61ee007b4246de0043612a691eb39e07efbcbb2b11be8e5", - Sha256: "cd00e292c5970d3c5e2f0ffa5171e555bc46bfc4faddfb4a418b6840b86e79a3", - } - - sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("kubectl-prefetch-proxy-bytes-100")) + sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("-100", testFile.GetInfo().Size())).To(Equal(sha256sum)) - time.Sleep(5 * time.Second) - sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + time.Sleep(1 * time.Second) + sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy-bytes-100"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=0-100", func() { + Context("1MiB file and set range header bytes=0-100", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download", "range: bytes=0-100"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 0-100 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-0-100' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-prefetch-proxy-bytes-0-100"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 0-100 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-0-100' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "eb46a424abb2151d4d1337d030f3d15b510cf7827a45a0e033e81ed3d15d8743", - Sha256: "7bbfcb694f6acc69483751d3b48d5bdbdcb284f9e04c7b7caa04b5977cffddc8", - } - - sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("kubectl-prefetch-proxy-bytes-0-100")) + sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("0-100", testFile.GetInfo().Size())).To(Equal(sha256sum)) - time.Sleep(5 * time.Second) - sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + time.Sleep(1 * time.Second) + sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy-bytes-0-100"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=100-", func() { + Context("1MiB file and set range header bytes=100-", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download", "range: bytes=100-"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100- -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100-' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-prefetch-proxy-bytes-100-"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100- -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100-' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "148e03a3ba5c45269de90465cff82d003e52a7ec7e3a2ea6fe08e45f4fb6fbdd", - Sha256: "ba9a10ceceb80562fc124dc9bc94ea2a38e3a71e3e746e2140e6381ac791cdeb", - } - - sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("kubectl-prefetch-proxy-bytes-100-")) + sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("100-", testFile.GetInfo().Size())).To(Equal(sha256sum)) - time.Sleep(5 * time.Second) - sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + time.Sleep(1 * time.Second) + sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy-bytes-100-"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=100-46026751", func() { - It("download should be ok", Label("prefetch-proxy", "download", "range: bytes=100-46026751"), func() { + Context("1MiB file and set range header bytes=100-1048575", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("download should be ok", Label("prefetch-proxy", "download", "range: bytes=100-1048575"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-46026751 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100-46026751' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-prefetch-proxy-bytes-100-46026751"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-1048575 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100-1048575' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "daf131380b88dd60a81140f87d36767cbfadc314324b71e0ee2a35679cf864b5", - Sha256: "ba9a10ceceb80562fc124dc9bc94ea2a38e3a71e3e746e2140e6381ac791cdeb", - } - - sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("kubectl-prefetch-proxy-bytes-100-46026751")) + sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("100-1048575", testFile.GetInfo().Size())).To(Equal(sha256sum)) - time.Sleep(5 * time.Second) - sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + time.Sleep(1 * time.Second) + sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy-bytes-100-1048575"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=0-46026751", func() { - It("download should be ok", Label("prefetch-proxy", "download", "range: bytes=0-46026751"), func() { + Context("1MiB file and set range header bytes=0-1048575", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("download should be ok", Label("prefetch-proxy", "download", "range: bytes=0-1048575"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 0-46026751 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-0-46026751' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-prefetch-proxy-bytes-0-46026751"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 0-1048575 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-0-1048575' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "90f143ba29e0174095ca2e59a76d86f3acaa4072aec3263c2bacf51259eaece0", - Sha256: "327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a", - } - - sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("kubectl-prefetch-proxy-bytes-0-46026751")) + sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("0-1048575", testFile.GetInfo().Size())).To(Equal(sha256sum)) - time.Sleep(5 * time.Second) - sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + time.Sleep(1 * time.Second) + sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy-bytes-0-1048575"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=100-10240", func() { + Context("1MiB file and set range header bytes=100-10240", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download", "range: bytes=100-10240"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-10240 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100-10240' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-prefetch-proxy-bytes-100-10240"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-10240 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100-10240' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "52c748b962167e66421259a7834363b3956b1a3c5f8cea5575a1f96dae4be47d", - Sha256: "8732360b941ad09a5e0e5d5f9891118bc068f6d0d5a56e3c6d483e4600fbc43f", - } - - sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("kubectl-prefetch-proxy-bytes-100-10240")) + sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("100-10240", testFile.GetInfo().Size())).To(Equal(sha256sum)) - time.Sleep(5 * time.Second) - sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + time.Sleep(1 * time.Second) + sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy-bytes-100-10240"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) - Context("/bin/kubectl file and set range header bytes=100-1024", func() { + Context("1MiB file and set range header bytes=100-1024", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("download should be ok", Label("prefetch-proxy", "download", "range: bytes=100-1024"), func() { seedClientPod, err := util.SeedClientExec(0) fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-1024 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100-1024' %s --output %s", util.GetFileURL("/bin/kubectl"), util.GetOutputPath("kubectl-prefetch-proxy-bytes-100-1024"))).CombinedOutput() + out, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("curl -x 127.0.0.1:4001 -r 100-1024 -H 'X-Dragonfly-Tag: prefetch-proxy-bytes-100-1024' %s --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) fmt.Println(string(out)) - fileMetadata := util.FileMetadata{ - ID: "52c748b962167e66421259a7834363b3956b1a3c5f8cea5575a1f96dae4be47d", - Sha256: "d6d17dca18b8de59e38da525dc24c47b74fec1a790a9f64afdd6538f4f8fa90e", - } - - sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, util.GetOutputPath("kubectl-prefetch-proxy-bytes-100-1024")) + sha256sum, err := util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetRangeSha256("100-1024", testFile.GetInfo().Size())).To(Equal(sha256sum)) - time.Sleep(5 * time.Second) - sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, fileMetadata.ID) + time.Sleep(1 * time.Second) + sha256sum, err = util.CalculateSha256ByTaskID([]*util.PodExec{seedClientPod}, testFile.GetTaskID(util.WithTaskIDTag("prefetch-proxy-bytes-100-1024"))) Expect(err).NotTo(HaveOccurred()) - Expect("327b4022d0bfd1d5e9c0701d4a3f989a536f7e6e865e102dcd77c7e7adb31f9a").To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) }) diff --git a/test/e2e/v2/util/dufs.go b/test/e2e/v2/util/dufs.go new file mode 100644 index 00000000000..29aefe5da5a --- /dev/null +++ b/test/e2e/v2/util/dufs.go @@ -0,0 +1,118 @@ +/* + * Copyright 2025 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "os" + "path/filepath" +) + +const ( + // defaultDufsFileServerEndpoint is the default endpoint of the dufs file server. + defaultDufsFileServerEndpoint = "http://dufs.dragonfly-e2e.svc" + + // defaultDufsFileServerLocalDir is the default local directory of the dufs file server. + defaultDufsFileServerLocalDir = "/tmp/artifact/dufs" +) + +// dufsFileServer represents a file server implemented by project dufs(https://github.com/sigoden/dufs). +type dufsFileServer struct { + // endpoint is the endpoint of the file server. + endpoint *url.URL + + // localDir is the local directory of the file server. + localDir string +} + +// newDufsFileServer creates a new file server. +func newDufsFileServer(opts ...FileServerOption) (*dufsFileServer, error) { + fileServerOptions := &fileServer{ + endpoint: defaultDufsFileServerEndpoint, + localDir: defaultDufsFileServerLocalDir, + } + + for _, opt := range opts { + opt(fileServerOptions) + } + + // parse the endpoint + var u *url.URL + u, err := url.Parse(fileServerOptions.endpoint) + if err != nil { + return nil, err + } + + // ensure the localDir exists + if err = os.MkdirAll(fileServerOptions.localDir, 0777); err != nil { + return nil, err + } + + fileServer := &dufsFileServer{ + endpoint: u, + localDir: fileServerOptions.localDir, + } + + return fileServer, nil +} + +// GenerateFile generates a file by the size and returns File instance. +func (fs *dufsFileServer) GenerateFile(size FileSize) (*File, error) { + // create file in the localDir + data := make([]byte, size) + _, err := rand.Read(data) + if err != nil { + return nil, err + } + + hash := sha256.Sum256(data) + fileName := hex.EncodeToString(hash[:]) + filePath := filepath.Join(fs.localDir, fileName) + if err = os.WriteFile(filePath, data, 0666); err != nil { + return nil, err + } + + // get file info + info, err := os.Stat(filePath) + if err != nil { + return nil, err + } + + file := &File{ + info: info, + localPath: filePath, + downloadURL: fmt.Sprintf("%s/%s", fs.endpoint.String(), fileName), + } + + return file, nil +} + +// DeleteFile deletes the file by the file info. +func (fs *dufsFileServer) DeleteFile(info os.FileInfo) error { + // delete the file in the local dir + filepath := filepath.Join(fs.localDir, info.Name()) + return os.RemoveAll(filepath) +} + +// Purge cleans up all files generated by the file server. +func (fs *dufsFileServer) Purge() error { + return os.RemoveAll(fs.localDir) +} diff --git a/test/e2e/v2/util/file.go b/test/e2e/v2/util/file.go new file mode 100644 index 00000000000..c529594ea84 --- /dev/null +++ b/test/e2e/v2/util/file.go @@ -0,0 +1,160 @@ +/* + * Copyright 2025 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "bufio" + "crypto/sha256" + "fmt" + "io" + "os" + "path" + "strconv" + "strings" + + "d7y.io/dragonfly/v2/pkg/idgen" + nethttp "d7y.io/dragonfly/v2/pkg/net/http" +) + +type FileSize = uint64 + +const ( + // FileSize1MiB represents the size of 1MiB. + FileSize1MiB FileSize = 1024 * 1024 + + // FileSize10MiB represents the size of 10MiB. + FileSize10MiB FileSize = 10 * FileSize1MiB + + // FileSize100MiB represents the size of 100MiB. + FileSize100MiB FileSize = 100 * FileSize1MiB + + // FileSize500MiB represents the size of 500MiB. + FileSize500MiB FileSize = 500 * FileSize1MiB + + // FileSize1GiB represents the size of 1GiB. + FileSize1GiB FileSize = 1024 * FileSize1MiB +) + +// File represents a file. +type File struct { + // info is the local file info. + info os.FileInfo + + // localPath is the local path of the file. + localPath string + + // downloadURL is the download URL of the file from remote file server. + downloadURL string +} + +// GetInfo returns the file info. +func (f *File) GetInfo() os.FileInfo { + return f.info +} + +// GetSha256 returns the sha256 of the file content. +func (f *File) GetSha256() string { + // the file name is the sha256 of the file content + return f.info.Name() +} + +// GetRangeSha256 returns the sha256 of the range part of the file content. +func (f *File) GetRangeSha256(r string, fileSize int64) string { + // parse the range header + parsedRange := nethttp.MustParseRange(fmt.Sprintf("bytes=%s", r), fileSize) + start, end := parsedRange.Start, parsedRange.Start+parsedRange.Length-1 + + file, err := os.Open(f.localPath) + if err != nil { + fmt.Printf("open file %s error: %v\n", f.localPath, err) + return "" + } + defer file.Close() + + if _, err = file.Seek(start, 0); err != nil { + fmt.Printf("seek file %s to %d error: %v\n", f.localPath, start, err) + return "" + } + + limitedReader := io.LimitReader(file, end-start+1) + bufferedReader := bufio.NewReader(limitedReader) + + hash := sha256.New() + if _, err = io.Copy(hash, bufferedReader); err != nil { + fmt.Printf("copy file %s content error: %v\n", f.localPath, err) + return "" + } + + return fmt.Sprintf("%x", hash.Sum(nil)) +} + +// GetDownloadURL returns the download URL of the file from remote file server. +func (f *File) GetDownloadURL() string { + return f.downloadURL +} + +// GetTaskID returns the task id of the file. +func (f *File) GetTaskID(opts ...TaskIDOption) string { + taskIDOptions := &taskID{ + url: f.downloadURL, + } + + for _, opt := range opts { + opt(taskIDOptions) + } + + return idgen.TaskIDV2(taskIDOptions.url, taskIDOptions.tag, taskIDOptions.application, taskIDOptions.filteredQueryParams) +} + +// GetOutputPath returns the output path of the file. +func (f *File) GetOutputPath() string { + return path.Join("/tmp", f.info.Name()) +} + +// parseRangeHeader parses the Range header value and returns start and end positions. +func parseRangeHeader(rangeHeader string, fileSize int64) (start, end int64) { + if rangeHeader == "" { + return 0, fileSize - 1 + } + + // split the range header by "-" + parts := strings.Split(rangeHeader, "-") + + // handle different range formats + switch { + case parts[0] == "": // -N: last N bytes + end = fileSize - 1 + bytes, _ := strconv.ParseInt(parts[1], 10, 64) + start = fileSize - bytes + if start < 0 { + start = 0 + } + + case parts[1] == "": // N-: from N to end + start, _ = strconv.ParseInt(parts[0], 10, 64) + end = fileSize - 1 + + default: // N-M: from N to M + start, _ = strconv.ParseInt(parts[0], 10, 64) + end, _ = strconv.ParseInt(parts[1], 10, 64) + if end >= fileSize { + end = fileSize - 1 + } + } + + return start, end +} diff --git a/test/e2e/v2/util/file_server.go b/test/e2e/v2/util/file_server.go index 7e6b7c17e56..f0bfd5a1029 100644 --- a/test/e2e/v2/util/file_server.go +++ b/test/e2e/v2/util/file_server.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Dragonfly Authors + * Copyright 2025 The Dragonfly Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,18 +16,68 @@ package util -import "fmt" +import ( + "os" + "sync" +) -type FileMetadata struct { - ID string - Sha256 string +var ( + // globalFileServer is the global file server instance. + globalFileServer FileServer + // once is the sync.Once instance for globalFileServer. + once sync.Once +) + +// NewFileServer creates a new file server which using the singleton pattern. +func NewFileServer() (FileServer, error) { + var err error + once.Do(func() { + // currently only support dufs file server, + // can be extended to support more file servers in the future. + globalFileServer, err = newDufsFileServer() + }) + + return globalFileServer, err +} + +// GetFileServer returns the global file server instance. +func GetFileServer() FileServer { + return globalFileServer } -func GetFileURL(filePath string) string { - baseURL := "http://file-server.dragonfly-e2e.svc/kind" - return fmt.Sprintf("%s%s", baseURL, filePath) +// FileServer is the interface of file server for testing. +type FileServer interface { + // GenerateFile generates a file by random data with the specified size. + GenerateFile(size FileSize) (*File, error) + + // DeleteFile deletes the file by the file info. + DeleteFile(info os.FileInfo) error + + // Purge deletes all files generated by the file server, it should be called after all tests. + Purge() error +} + +// fileServer represents the config of the file server. +type fileServer struct { + // endpoint is the endpoint of the file server. + endpoint string + // localDir is the local temporary directory of the file server. + localDir string +} + +// FileServerOption is the type of the options of the file server. +type FileServerOption func(*fileServer) + +// WithFileServerEndpoint sets the endpoint of the file server. +func WithFileServerEndpoint(endpoint string) FileServerOption { + return func(o *fileServer) { + o.endpoint = endpoint + } } -func GetOutputPath(fileName string) string { - return fmt.Sprintf("/tmp/%s", fileName) +// WithFileServerLocalDir sets the local temporary directory of the file server. +func WithFileServerLocalDir(localDir string) FileServerOption { + return func(o *fileServer) { + o.localDir = localDir + } } diff --git a/test/e2e/v2/util/task.go b/test/e2e/v2/util/task.go index 8fcbee350b2..71a1f3f9994 100644 --- a/test/e2e/v2/util/task.go +++ b/test/e2e/v2/util/task.go @@ -99,3 +99,46 @@ func CalculateSha256ByOutput(pods []*PodExec, output string) (string, error) { return sha256sum, nil } + +// taskID represents the config of the task id. +type taskID struct { + // url is the url of the download task. + url string + // tag is the tag of the download task. + tag string + // appliccation is the application of the download task. + application string + // filteredQueryParams is the filtered query params of the download task. + filteredQueryParams []string +} + +// TaskIDOption is the type of the options of the task id. +type TaskIDOption func(*taskID) + +// WithTaskIDURL sets the url of the download task. +func WithTaskIDURL(url string) TaskIDOption { + return func(o *taskID) { + o.url = url + } +} + +// WithTaskIDTag sets the tag of the download task. +func WithTaskIDTag(tag string) TaskIDOption { + return func(o *taskID) { + o.tag = tag + } +} + +// WithTaskIDApplication sets the application of the download task. +func WithTaskIDApplication(application string) TaskIDOption { + return func(o *taskID) { + o.application = application + } +} + +// WithTaskIDFilteredQueryParams sets the filtered query params of the download task. +func WithTaskIDFilteredQueryParams(filteredQueryParams []string) TaskIDOption { + return func(o *taskID) { + o.filteredQueryParams = filteredQueryParams + } +} diff --git a/test/testdata/k8s/dufs.yaml b/test/testdata/k8s/dufs.yaml new file mode 100644 index 00000000000..d8aea6aba0d --- /dev/null +++ b/test/testdata/k8s/dufs.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: dragonfly-e2e + +--- +apiVersion: v1 +kind: Service +metadata: + name: dufs + namespace: dragonfly-e2e +spec: + selector: + app: dragonfly + component: dufs + type: ClusterIP + ports: + - name: server + port: 80 + protocol: TCP + targetPort: 5000 + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: dufs + namespace: dragonfly-e2e +spec: + serviceName: dufs + selector: + matchLabels: + app: dragonfly + component: dufs + replicas: 1 + template: + metadata: + labels: + app: dragonfly + component: dufs + spec: + containers: + - name: dufs + image: sigoden/dufs:v0.43.0 + command: ["dufs"] + args: + - "/data" + - "-A" + - "-b" + - "0.0.0.0" + - "-p" + - "5000" + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5000 + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + hostPath: + path: /tmp/artifact/dufs