feat: import data command for bitbucket cloud (#266)

* feat: import:data for Bitbucket Cloud Add support for generating import targets data file for Bitbucket Cloud by listing existing repos in users Bitbucket Cloud. * refactor: add return request types Co-authored-by: ghe <[email protected]>
snyk · Feb 14, 2022 · 83772d0 · 83772d0
1 parent 616d43b
commit 83772d0
Show file tree

Hide file tree

Showing 24 changed files with 428 additions and 67 deletions.
diff --git a/src/cmds/import:data.ts b/src/cmds/import:data.ts
@@ -46,6 +46,7 @@ const entityName: {
   gitlab: 'group',
   'azure-repos': 'org',
   'bitbucket-server': 'project',
+  'bitbucket-cloud': 'workspace',
 };
 
 export async function handler(argv: {

diff --git a/src/cmds/orgs:data.ts b/src/cmds/orgs:data.ts
@@ -39,7 +39,7 @@ export const builder = {
     default: SupportedIntegrationTypesImportOrgData.GITHUB,
     choices: [...Object.values(SupportedIntegrationTypesImportOrgData)],
     desc:
-      'The source of the targets to be imported e.g. Github, Github Enterprise',
+      'The source of the targets to be imported e.g. Github, Github Enterprise, Gitlab, Bitbucket Server',
   },
 };
 

diff --git a/src/lib/index.ts b/src/lib/index.ts
@@ -15,3 +15,4 @@ export * from './source-handlers/github';
 export * from './source-handlers/gitlab';
 export * from './source-handlers/azure';
 export * from './source-handlers/bitbucket-server';
+export * from './source-handlers/bitbucket-cloud';
diff --git a/src/lib/request-with-rate-limit.ts b/src/lib/request-with-rate-limit.ts
@@ -5,14 +5,22 @@ import { OutgoingHttpHeaders } from 'http2';
 
 const debug = debugLib('snyk:limiter');
 
-export async function limiterWithRateLimitRetries(
+export async function limiterWithRateLimitRetries<ResponseType>(
   verb: needle.NeedleHttpVerbs,
   url: string,
   headers: OutgoingHttpHeaders,
   limiter: Bottleneck,
   rateLimitSleepTime: number,
-): Promise<any> {
-  let data;
+): Promise<{
+  statusCode: number;
+  body: ResponseType;
+  headers: Record<string, unknown>;
+}> {
+  let data: {
+    statusCode: number;
+    body: ResponseType;
+    headers: Record<string, unknown>;
+  };
   const maxRetries = 7;
   let attempt = 0;
   limiter.on('failed', async (error, jobInfo) => {
@@ -25,15 +33,21 @@ export async function limiterWithRateLimitRetries(
     }
   });
   while (attempt < maxRetries) {
-    data = await limiter.schedule(() =>
+    data = (await limiter.schedule(() =>
       needle(verb, url, { headers: headers }),
-    );
+    )) as {
+      statusCode: number;
+      body: ResponseType;
+      headers: Record<string, unknown>;
+    };
     if ([404, 200].includes(Number(data.statusCode))) {
       break;
     }
     if (data.statusCode === 401) {
       console.error(
-        `ERROR: ${data.body}. Please check the token and try again.`,
+        `ERROR: ${JSON.stringify(
+          data.body,
+        )}. Please check the token and try again.`,
       );
       break;
     }
@@ -46,5 +60,6 @@ export async function limiterWithRateLimitRetries(
     }
     attempt += 1;
   }
-  return data;
+
+  return data!;
 }
diff --git a/src/lib/source-handlers/azure/list-projects.ts b/src/lib/source-handlers/azure/list-projects.ts
@@ -2,20 +2,23 @@ import * as qs from 'querystring';
 import base64 = require('base-64');
 import * as debugLib from 'debug';
 import { OutgoingHttpHeaders } from 'http2';
-import { AzureProjectData } from './types';
 import { getAzureToken } from './get-azure-token';
 import { getBaseUrl } from './get-base-url';
-import { AzureProjectData as Project } from './types';
+import { AzureProjectData } from './types';
 import { limiterWithRateLimitRetries } from '../../request-with-rate-limit';
 import { limiterForScm } from '../../limiters';
 
 const debug = debugLib('snyk:azure');
 
+interface AzureProjectsResponse {
+  value: AzureProjectData[];
+}
+
 async function fetchAllProjects(
   orgName: string,
   host: string,
-): Promise<Project[]> {
-  const projectList: Project[] = [];
+): Promise<AzureProjectData[]> {
+  const projectList: AzureProjectData[] = [];
   let hasMorePages = true;
   let displayPageNumber = 1;
   let nextPage: string | undefined;
@@ -51,7 +54,7 @@ async function getProjects(
   host: string,
   continueFrom?: string,
 ): Promise<{
-  projects: Project[];
+  projects: AzureProjectData[];
   continueFrom?: string;
 }> {
   const azureToken = getAzureToken();
@@ -65,22 +68,26 @@ async function getProjects(
     Authorization: `Basic ${base64.encode(':' + azureToken)}`,
   };
   const limiter = await limiterForScm(1, 500);
-  const data = await limiterWithRateLimitRetries(
+  const {
+    body,
+    statusCode,
+    headers: responseHeaders,
+  } = await limiterWithRateLimitRetries<AzureProjectsResponse>(
     'get',
     `${host}/${orgName}/_apis/projects?${query}`,
     headers,
     limiter,
     60000,
   );
-  if (data.statusCode != 200) {
+  if (statusCode != 200) {
     throw new Error(`Failed to fetch projects for ${host}/${orgName}/_apis/projects?${query}\n
-    Status Code: ${data.statusCode}\n
-    Response body: ${JSON.stringify(data.body)}`);
+    Status Code: ${statusCode}\n
+    Response body: ${JSON.stringify(body)}`);
   }
-  const { value: projects } = data.body;
+  const { value: projects } = body;
   return {
     projects,
-    continueFrom: data.headers['x-ms-continuationtoken'] as string,
+    continueFrom: responseHeaders['x-ms-continuationtoken'] as string,
   };
 }
 

diff --git a/src/lib/source-handlers/azure/list-repos.ts b/src/lib/source-handlers/azure/list-repos.ts
@@ -10,6 +10,13 @@ import { limiterForScm } from '../../limiters';
 
 const debug = debugLib('snyk:azure');
 
+interface AzureReposResponse {
+  value: {
+    name: string;
+    project: { name: string };
+    defaultBranch: string;
+  }[];
+}
 export async function fetchAllRepos(
   url: string,
   orgName: string,
@@ -37,7 +44,9 @@ async function getRepos(
     Authorization: `Basic ${base64.encode(':' + token)}`,
   };
   const limiter = await limiterForScm(1, 500);
-  const data = await limiterWithRateLimitRetries(
+  const { body, statusCode } = await limiterWithRateLimitRetries<
+    AzureReposResponse
+  >(
     'get',
     `${url}/${orgName}/` +
       encodeURIComponent(project) +
@@ -46,30 +55,24 @@ async function getRepos(
     limiter,
     60000,
   );
-  if (data.statusCode != 200) {
+  if (statusCode != 200) {
     throw new Error(`Failed to fetch repos for ${url}/${orgName}/${encodeURIComponent(
       project,
     )}/_apis/git/repositories?api-version=4.1\n
-    Status Code: ${data.statusCode}\n
-    Response body: ${JSON.stringify(data.body)}`);
+    Status Code: ${statusCode}\n
+    Response body: ${JSON.stringify(body)}`);
   }
-  const repos = data.body['value'];
-  repos.map(
-    (repo: {
-      name: string;
-      project: { name: string };
-      defaultBranch: string;
-    }) => {
-      const { name, project, defaultBranch } = repo;
-      if (name && project && project.name) {
-        repoList.push({
-          name,
-          owner: project.name,
-          branch: defaultBranch || '',
-        });
-      }
-    },
-  );
+  const { value: repos } = body;
+  repos.map((repo) => {
+    const { name, project, defaultBranch } = repo;
+    if (name && project && project.name && defaultBranch) {
+      repoList.push({
+        name,
+        owner: project.name,
+        branch: defaultBranch,
+      });
+    }
+  });
   return repoList;
 }
 

diff --git a/src/lib/source-handlers/bitbucket-cloud/get-bitbucket-cloud-password.ts b/src/lib/source-handlers/bitbucket-cloud/get-bitbucket-cloud-password.ts
@@ -0,0 +1,9 @@
+export function getBitbucketCloudPassword(): string {
+  const bitbucketCloudPassword = process.env.BITBUCKET_CLOUD_PASSWORD;
+  if (!bitbucketCloudPassword) {
+    throw new Error(
+      `Please set the BITBUCKET_CLOUD_PASSWORD e.g. export BITBUCKET_CLOUD_PASSWORD='mybitbucketCloudPassword'`,
+    );
+  }
+  return bitbucketCloudPassword;
+}
diff --git a/src/lib/source-handlers/bitbucket-cloud/get-bitbucket-cloud-username.ts b/src/lib/source-handlers/bitbucket-cloud/get-bitbucket-cloud-username.ts
@@ -0,0 +1,9 @@
+export function getBitbucketCloudUsername(): string {
+  const bitbucketCLoudUsername = process.env.BITBUCKET_CLOUD_USERNAME;
+  if (!bitbucketCLoudUsername) {
+    throw new Error(
+      `Please set the BITBUCKET_CLOUD_USERNAME e.g. export BITBUCKET_CLOUD_USERNAME='myBitbucketCloudUsername'`,
+    );
+  }
+  return bitbucketCLoudUsername;
+}
diff --git a/src/lib/source-handlers/bitbucket-cloud/index.ts b/src/lib/source-handlers/bitbucket-cloud/index.ts
@@ -0,0 +1,4 @@
+export * from './list-workspaces';
+export * from './list-repos';
+export * from './workspace-is-empty';
+export * from './types';
diff --git a/src/lib/source-handlers/bitbucket-cloud/list-repos.ts b/src/lib/source-handlers/bitbucket-cloud/list-repos.ts
@@ -0,0 +1,114 @@
+import * as debugLib from 'debug';
+import base64 = require('base-64');
+import { OutgoingHttpHeaders } from 'http2';
+import { BitbucketCloudRepoData } from './types';
+import { getBitbucketCloudUsername } from './get-bitbucket-cloud-username';
+import { getBitbucketCloudPassword } from './get-bitbucket-cloud-password';
+import { limiterForScm } from '../../limiters';
+import { limiterWithRateLimitRetries } from '../../request-with-rate-limit';
+
+const debug = debugLib('snyk:bitbucket-cloud');
+
+interface BitbucketReposResponse {
+  values: {
+    mainbranch: {
+      name: string;
+    };
+    slug: string;
+    workspace: {
+      slug: string;
+      uuid: string;
+    };
+  }[];
+  next?: string;
+}
+
+export const fetchAllBitbucketCloudRepos = async (
+  workspace: string,
+  username: string,
+  password: string,
+): Promise<BitbucketCloudRepoData[]> => {
+  let lastPage = false;
+  let reposList: BitbucketCloudRepoData[] = [];
+  let pageCount = 1;
+  let nextPage;
+  while (!lastPage) {
+    debug(`Fetching page ${pageCount} for ${workspace}\n`);
+    try {
+      const {
+        repos,
+        next,
+      }: { repos: BitbucketCloudRepoData[]; next?: string } = await getRepos(
+        workspace,
+        username,
+        password,
+        nextPage,
+      );
+
+      reposList = reposList.concat(repos);
+      next
+        ? ((lastPage = false), (nextPage = next))
+        : ((lastPage = true), (nextPage = ''));
+      pageCount++;
+    } catch (err) {
+      throw new Error(JSON.stringify(err));
+    }
+  }
+  return reposList;
+};
+
+const getRepos = async (
+  workspace: string,
+  username: string,
+  password: string,
+  nextPageLink?: string,
+): Promise<{ repos: BitbucketCloudRepoData[]; next?: string }> => {
+  const repos: BitbucketCloudRepoData[] = [];
+  const headers: OutgoingHttpHeaders = {
+    Authorization: `Basic ${base64.encode(username + ':' + password)}`,
+  };
+  const limiter = await limiterForScm(1, 1000, 1000, 1000, 1000 * 3600);
+  const { statusCode, body } = await limiterWithRateLimitRetries<
+    BitbucketReposResponse
+  >(
+    'get',
+    nextPageLink ?? `https://bitbucket.org/api/2.0/repositories/${workspace}`,
+    headers,
+    limiter,
+    60000,
+  );
+  if (statusCode != 200) {
+    throw new Error(`Failed to fetch projects for ${
+      nextPageLink != ''
+        ? nextPageLink
+        : `https://bitbucket.org/api/2.0/repositories/${workspace}`
+    }\n
+      Status Code: ${statusCode}\n
+      Response body: ${JSON.stringify(body)}`);
+  }
+  const { next, values } = body;
+  for (const repo of values) {
+    const { workspace, mainbranch, slug } = repo;
+    if (mainbranch.name && workspace && slug)
+      repos.push({
+        owner: workspace.slug ? workspace.slug : repo.workspace.uuid,
+        name: slug,
+        branch: mainbranch.name,
+      });
+  }
+  return { repos, next };
+};
+
+export async function listBitbucketCloudRepos(
+  workspace: string,
+): Promise<BitbucketCloudRepoData[]> {
+  const bitbucketCloudUsername = getBitbucketCloudUsername();
+  const bitbucketCloudPassword = getBitbucketCloudPassword();
+  debug(`Fetching all repos data for org: ${workspace}`);
+  const repoList = await fetchAllBitbucketCloudRepos(
+    workspace,
+    bitbucketCloudUsername,
+    bitbucketCloudPassword,
+  );
+  return repoList;
+}