Merge pull request #899 from internxt/feature/PB-1063-add-web-workers…

…-to-upload

[PB-1063] feature/add web workers to upload

CODEOWNERS

@@ -4,3 +4,4 @@
 /src/app/crypto/ @sg-gs
 /src/app/core/services/stream.service.ts @sg-gs
 /src/app/crypto/services/pgp.service.ts @sg-gs
+/src/upload.worker.ts @sg-gs

babel.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  presets: ['@babel/preset-env', '@babel/preset-react'],
+  plugins: [['react-refresh/babel', { skipEnvCheck: true }], '@babel/plugin-proposal-class-properties'],
+};

package.json

@@ -113,6 +113,7 @@
     ]
   },
   "devDependencies": {
+    "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
     "@craco/craco": "^7.1.0",
     "@internxt/eslint-config-internxt": "^1.0.3",
     "@internxt/prettier-config": "^1.0.1",

src/WebWorker.ts

@@ -0,0 +1,3 @@
+export const createUploadWebWorker = (): Worker => {
+  return new Worker(new URL('./upload.worker', import.meta.url), { type: 'module' });
+};

src/app/backups/views/BackupsView/BackupsView.tsx

@@ -1,6 +1,4 @@
 import { useEffect, useState } from 'react';
-import UilHdd from '@iconscout/react-unicons/icons/uil-hdd';
-
 import DeviceList from '../../components/DeviceList/DeviceList';
 import { Device } from '../../types';
 import Breadcrumbs, { BreadcrumbItemData } from 'app/shared/components/Breadcrumbs/Breadcrumbs';

src/app/core/services/stream.service.ts

@@ -1,7 +1,3 @@
-import createZipReadable from 'app/drive/services/download.service/downloadFolder/zipStream';
-import streamSaver from 'streamsaver';
-import { loadWritableStreamPonyfill } from 'app/network/download';
-
 type BinaryStream = ReadableStream<Uint8Array>;
 
 export async function binaryStreamToBlob(stream: BinaryStream, mimeType?: string): Promise<Blob> {
@@ -134,93 +130,3 @@ export function streamFileIntoChunks(
     },
   });
 }
-
-type FlatFolderZipOpts = {
-  abortController?: AbortController;
-  progress?: (loadedBytes: number) => void;
-};
-
-type AddFileToZipFunction = (name: string, source: ReadableStream<Uint8Array>) => void;
-type AddFolderToZipFunction = (name: string) => void;
-
-interface ZipStream {
-  addFile: AddFileToZipFunction;
-  addFolder: AddFolderToZipFunction;
-  stream: ReadableStream<Uint8Array>;
-  end: () => void;
-}
-
-export class FlatFolderZip {
-  private finished!: Promise<void>;
-  private zip: ZipStream;
-  private abortController?: AbortController;
-
-  constructor(folderName: string, opts: FlatFolderZipOpts) {
-    this.zip = createFolderWithFilesWritable();
-    this.abortController = opts.abortController;
-
-    const passThrough = opts.progress ? buildProgressStream(this.zip.stream, opts.progress) : this.zip.stream;
-
-    const isFirefox = navigator.userAgent.indexOf('Firefox') != -1;
-
-    if (isFirefox) {
-      loadWritableStreamPonyfill().then(() => {
-        streamSaver.WritableStream = window.WritableStream;
-
-        this.finished = passThrough.pipeTo(streamSaver.createWriteStream(folderName + '.zip'), {
-          signal: opts.abortController?.signal,
-        });
-      });
-    } else {
-      this.finished = passThrough.pipeTo(streamSaver.createWriteStream(folderName + '.zip'), {
-        signal: opts.abortController?.signal,
-      });
-    }
-  }
-
-  addFile(name: string, source: ReadableStream<Uint8Array>): void {
-    if (this.abortController?.signal.aborted) return;
-
-    this.zip.addFile(name, source);
-  }
-
-  addFolder(name: string): void {
-    if (this.abortController?.signal.aborted) return;
-
-    this.zip.addFolder(name);
-  }
-
-  async close(): Promise<void> {
-    if (this.abortController?.signal.aborted) return;
-
-    this.zip.end();
-    await this.finished;
-  }
-
-  abort(): void {
-    this.abortController?.abort();
-  }
-}
-
-function createFolderWithFilesWritable(): ZipStream {
-  let controller;
-
-  const zipStream = createZipReadable({
-    start(ctrl) {
-      controller = ctrl;
-    },
-  });
-
-  return {
-    addFile: (name: string, source: ReadableStream<Uint8Array>): void => {
-      controller.enqueue({ name, stream: () => source });
-    },
-    addFolder: (name: string): void => {
-      controller.enqueue({ name, directory: true });
-    },
-    stream: zipStream,
-    end: () => {
-      controller.close();
-    },
-  };
-}

src/app/core/services/zipFolder.service.ts

@@ -0,0 +1,93 @@
+import streamSaver from 'streamsaver';
+import { loadWritableStreamPonyfill } from 'app/network/download';
+import createZipReadable from 'app/drive/services/download.service/downloadFolder/zipStream';
+import { buildProgressStream } from './stream.service';
+
+type FlatFolderZipOpts = {
+  abortController?: AbortController;
+  progress?: (loadedBytes: number) => void;
+};
+
+type AddFileToZipFunction = (name: string, source: ReadableStream<Uint8Array>) => void;
+type AddFolderToZipFunction = (name: string) => void;
+
+interface ZipStream {
+  addFile: AddFileToZipFunction;
+  addFolder: AddFolderToZipFunction;
+  stream: ReadableStream<Uint8Array>;
+  end: () => void;
+}
+
+function createFolderWithFilesWritable(): ZipStream {
+  let controller;
+
+  const zipStream = createZipReadable({
+    start(ctrl) {
+      controller = ctrl;
+    },
+  });
+
+  return {
+    addFile: (name: string, source: ReadableStream<Uint8Array>): void => {
+      controller.enqueue({ name, stream: () => source });
+    },
+    addFolder: (name: string): void => {
+      controller.enqueue({ name, directory: true });
+    },
+    stream: zipStream,
+    end: () => {
+      controller.close();
+    },
+  };
+}
+
+export class FlatFolderZip {
+  private finished!: Promise<void>;
+  private zip: ZipStream;
+  private abortController?: AbortController;
+
+  constructor(folderName: string, opts: FlatFolderZipOpts) {
+    this.zip = createFolderWithFilesWritable();
+    this.abortController = opts.abortController;
+
+    const passThrough = opts.progress ? buildProgressStream(this.zip.stream, opts.progress) : this.zip.stream;
+
+    const isFirefox = navigator.userAgent.indexOf('Firefox') != -1;
+
+    if (isFirefox) {
+      loadWritableStreamPonyfill().then(() => {
+        streamSaver.WritableStream = window.WritableStream;
+        this.finished = passThrough.pipeTo(streamSaver.createWriteStream(folderName + '.zip'), {
+          signal: opts.abortController?.signal,
+        });
+      });
+    } else {
+      this.finished = passThrough.pipeTo(streamSaver.createWriteStream(folderName + '.zip'), {
+        signal: opts.abortController?.signal,
+      });
+    }
+  }
+
+  addFile(name: string, source: ReadableStream<Uint8Array>): void {
+    if (this.abortController?.signal.aborted) return;
+
+    this.zip.addFile(name, source);
+  }
+
+  addFolder(name: string): void {
+    if (this.abortController?.signal.aborted) return;
+
+    this.zip.addFolder(name);
+  }
+
+  async close(): Promise<void> {
+    if (this.abortController?.signal.aborted) return;
+
+    this.zip.end();
+    await this.finished;
+  }
+
+  abort(): void {
+    this.abortController?.abort();
+  }
+}

src/app/drive/services/download.service/downloadFolder/downloadFolderUsingFileSystemAccessAPI.ts

@@ -3,9 +3,8 @@ import { items } from '@internxt/lib';
 import { getEnvironmentConfig } from '../../network.service';
 import { DriveFolderData, FolderTree } from '../../../types';
 import folderService from '../../folder.service';
-import { FlatFolderZip } from 'app/core/services/stream.service';
+import { FlatFolderZip } from 'app/core/services/zipFolder.service';
 import network from 'app/network';
-import analyticsService from 'app/analytics/services/analytics.service';
 
 /**
  * @description Downloads a folder using File System Access API

src/app/drive/services/file.service/uploadFile.ts

@@ -9,7 +9,7 @@ import { getEnvironmentConfig } from '../network.service';
 import { encryptFilename } from '../../../crypto/services/utils';
 import errorService from '../../../core/services/error.service';
 import { SdkFactory } from '../../../core/factory/sdk';
-import { uploadFile as uploadToBucket } from 'app/network/upload';
+import { Network } from 'app/drive/services/network.service';
 import notificationsService, { ToastType } from '../../../notifications/services/notifications.service';
 import { generateThumbnailFromFile } from '../thumbnail.service';
 
@@ -21,23 +21,28 @@ export interface FileToUpload {
   parentFolderId: number;
 }
 
-export async function uploadFile(
-  userEmail: string,
-  file: FileToUpload,
-  isTeam: boolean,
-  updateProgressCallback: (progress: number) => void,
-  trackingParameters: { isMultipleUpload: 0 | 1; processIdentifier: string },
-  abortController?: AbortController,
+export interface FileUploadOptions {
+  isTeam: boolean;
+  trackingParameters: { isMultipleUpload: 0 | 1; processIdentifier: string };
+  abortController?: AbortController;
   ownerUserAuthenticationData?: {
     token: string;
     bridgeUser: string;
     bridgePass: string;
     encryptionKey: string;
     bucketId: string;
-  },
+  };
+  abortCallback?: (abort?: () => void) => void;
+}
+
+export async function uploadFile(
+  userEmail: string,
+  file: FileToUpload,
+  updateProgressCallback: (progress: number) => void,
+  options: FileUploadOptions,
 ): Promise<DriveFileData> {
   const { bridgeUser, bridgePass, encryptionKey, bucketId } =
-    ownerUserAuthenticationData ?? getEnvironmentConfig(isTeam);
+    options.ownerUserAuthenticationData ?? getEnvironmentConfig(options.isTeam);
   const isBrave = !!(navigator.brave && (await navigator.brave.isBrave()));
 
   const trackingUploadProperties: TrackingPlan.UploadProperties = {
@@ -48,8 +53,8 @@ export async function uploadFile(
     file_name: file.name,
     bandwidth: 0,
     band_utilization: 0,
-    process_identifier: trackingParameters?.processIdentifier,
-    is_multiple: trackingParameters?.isMultipleUpload,
+    process_identifier: options.trackingParameters?.processIdentifier,
+    is_multiple: options.trackingParameters?.isMultipleUpload,
     is_brave: isBrave,
   };
   try {
@@ -70,20 +75,18 @@ export async function uploadFile(
       throw new Error('Bucket not found!');
     }
 
-    const fileId = await uploadToBucket(bucketId, {
-      creds: {
-        pass: bridgePass,
-        user: bridgeUser,
-      },
+    const [promise, abort] = new Network(bridgeUser, bridgePass, encryptionKey).uploadFile(bucketId, {
       filecontent: file.content,
       filesize: file.size,
-      mnemonic: encryptionKey,
-      progressCallback: (totalBytes, uploadedBytes) => {
-        updateProgressCallback(uploadedBytes / totalBytes);
+      progressCallback: (progress) => {
+        updateProgressCallback(progress);
       },
-      abortController,
     });
 
+    options.abortCallback?.(abort?.abort);
+
+    const fileId = await promise;
+
     const name = encryptFilename(file.name, file.parentFolderId);
 
     const storageClient = SdkFactory.getInstance().createStorageClient();
@@ -98,15 +101,15 @@ export async function uploadFile(
       encrypt_version: StorageTypes.EncryptionVersion.Aes03,
     };
 
-    let response = await storageClient.createFileEntry(fileEntry, ownerUserAuthenticationData?.token);
+    let response = await storageClient.createFileEntry(fileEntry, options.ownerUserAuthenticationData?.token);
     if (!response.thumbnails) {
       response = {
         ...response,
         thumbnails: [],
       };
     }
 
-    const generatedThumbnail = await generateThumbnailFromFile(file, response.id, userEmail, isTeam);
+    const generatedThumbnail = await generateThumbnailFromFile(file, response.id, userEmail, options.isTeam);
     if (generatedThumbnail && generatedThumbnail.thumbnail) {
       response.thumbnails.push(generatedThumbnail.thumbnail);
       if (generatedThumbnail.thumbnailFile) {
@@ -125,7 +128,7 @@ export async function uploadFile(
   } catch (err: unknown) {
     const castedError = errorService.castError(err);
 
-    if (!abortController?.signal.aborted) {
+    if (!options.abortController?.signal.aborted) {
       analyticsService.trackFileUploadError({
         ...trackingUploadProperties,
         bucket_id: parseInt(bucketId),