Merge pull request #269 from snyk-tech-services/chore/update-test-and…

…-format-settings-in-packages-file

fix: enable tests & formatting

package.json

@@ -7,11 +7,11 @@
   },
   "scripts": {
     "format:check": "prettier --check '{''{lib,test}/!(test/**/fixtures)/**/*,*}.{js,ts,json,yml}'",
-    "format": "prettier --write '{''{src,test}/!(test/**/fixtures)/**/*,*}.{js,ts,json,yml}'",
+    "format": "prettier --write '{''{src,test}/!(fixtures)/**/*,*}.{js,ts,json,yml}'",
     "lint": "npm run format:check && npm run lint:eslint",
     "lint:eslint": "eslint --cache '{lib,test}/**/*.ts'",
     "test": "npm run lint && npm run build && npm run test:unit",
-    "test:unit": "jest --runInBand --ci --silent --clearCache",
+    "test:unit": "jest --runInBand --silent --ci",
     "test:unit:debug": "npm run build && DEBUG=* jest",
     "test:coverage": "npm run test:unit -- --coverage",
     "test:watch": "tsc-watch --onSuccess 'npm run test:unit'",

src/cmds/orgs:create.ts

@@ -48,7 +48,7 @@ export async function handler(argv: {
   } catch (e) {
     debug('Failed to create organizations.\n' + e);
     console.error(
-      `ERROR! Failed to create organizations. Try running with \`DEBUG=snyk* <command> for more info\`.\nERROR: ${e}`,
+      `${e}.\nTry running with \`DEBUG=snyk* <command> for more info\`.\nERROR: ${e}`,
     );
   }
 }

src/lib/api/poll-import/index.ts

@@ -118,7 +118,7 @@ export async function pollImportUrls(
         await logImportedProjects(locationUrl, projects);
         projectsArray.push(...projects);
       } catch (error) {
-        logFailedPollUrls(locationUrl, {
+        await logFailedPollUrls(locationUrl, {
           errorMessage:
             _.get(error, 'innerError.message') ||
             error.innerError ||

src/lib/source-handlers/github/index.ts

@@ -1,4 +1,4 @@
 export * from './list-organizations';
-export { listGithubRepos } from './list-repos';
+export * from './list-repos';
 export * from './organization-is-empty';
 export * from './types';

src/loggers/log-failed-org.ts

@@ -24,7 +24,7 @@ export async function logFailedOrg(
   });
 
   try {
-    log.info({ origName, groupId, errorMessage }, 'Failed to create org');
+    log.error({ origName, groupId, errorMessage }, 'Failed to create org');
   } catch (e) {
     debug('Failed to log failed organizations at location: ', e);
     // do nothing

src/scripts/create-orgs.ts

@@ -44,7 +44,8 @@ async function createNewOrgs(
     const { name, sourceOrgId } = orgData;
     try {
       debug(`Creating new "${name}" organization`);
-      const org = await createOrg(requestManager, groupId, name, sourceOrgId);      debug(`Creating new "${name}" organization`);
+      const org = await createOrg(requestManager, groupId, name, sourceOrgId);
+      debug(`Creating new "${name}" organization`);
       debug(`Listing integrations for new "${name}" organization`);
       const integrations =
         (await listIntegrations(requestManager, org.id)) || {};
@@ -62,7 +63,7 @@ async function createNewOrgs(
     } catch (e) {
       failed.push({ groupId, name, sourceOrgId });
       const errorMessage = e.data ? e.data.message : e.message;
-      logFailedOrg(
+      await logFailedOrg(
         groupId,
         name,
         errorMessage || 'Failed to create org, please try again in DEBUG mode.',
@@ -151,27 +152,43 @@ export async function createOrgs(
   });
 
   const createdOrgs: NewOrExistingOrg[] = [];
-  const existingOrgs: Org[] = [];
+  const allExistingOrgs: Org[] = [];
   const requestManager = new requestsManager({
     userAgentPrefix: 'snyk-api-import',
   });
 
   for (const groupId in orgsPerGroup) {
     let orgsToCreate = orgsPerGroup[groupId];
     debug(`Finding existing organizations in group ${groupId}`);
-    const res = await filterOutExistingOrgs(requestManager, orgsData, groupId);
-    existingOrgs.push(...res.existingOrgs);
+
+    const { newOrgs, existingOrgs } = await separateExistingOrganizations(
+      loggingPath,
+      requestManager,
+      groupId,
+      orgsToCreate,
+    );
+    allExistingOrgs.push(...existingOrgs);
     debug(`Found ${existingOrgs.length} existing organizations`);
 
     if (noDuplicateNames) {
-      orgsToCreate = res.newOrgs;
-      failedOrgs.push(
-        ...res.existingOrgs.map((o) => ({
-          groupId: o.group.id,
-          name: o.name,
-        })),
+      let duplicates = 0;
+      const uniqueOrgNames: Set<string> = new Set(
+        existingOrgs.map((org) => org.name),
       );
-      debug(`Found ${failedOrgs.length} duplicate organizations`);
+      for (const org of orgsToCreate) {
+        const { name } = org;
+        if (uniqueOrgNames.has(name)) {
+          duplicates += 1;
+          failedOrgs.push(org);
+          await logFailedOrg(
+            groupId,
+            name,
+            'Refusing to create a duplicate organization with option --noDuplicateNames enabled.',
+          );
+        }
+      }
+      debug(`Skipping ${duplicates} duplicate organizations`);
+      orgsToCreate = newOrgs;
     }
     debug(`Creating ${orgsToCreate.length} new organizations`);
     const { failed, created } = await createNewOrgs(
@@ -192,19 +209,50 @@ export async function createOrgs(
       )}/<groupId>.${FAILED_ORG_LOG_NAME}`,
     );
   }
-  debug(`Getting existing ${existingOrgs.length} orgs data`);
-  const { existing } = await listExistingOrgsData(requestManager, existingOrgs);
-  debug('Saving results');
+  debug(`Getting existing ${allExistingOrgs.length} orgs data`);
   const allOrgs: Partial<NewOrExistingOrg>[] = [...createdOrgs];
+  let existing: Partial<NewOrExistingOrg>[] = [];
+
   if (includeExistingOrgsInOutput) {
+    const res = await listExistingOrgsData(requestManager, allExistingOrgs);
+    existing = res.existing;
     allOrgs.push(...existing);
   }
   const fileName = await saveCreatedOrgData(allOrgs);
   return {
     orgs: createdOrgs,
-    existing: includeExistingOrgsInOutput ? existing : [],
+    existing,
     failed: failedOrgs,
     fileName,
     totalOrgs: orgsData.length,
   };
 }
+
+async function separateExistingOrganizations(
+  loggingPath: string,
+  requestManager: requestsManager,
+  groupId: string,
+  orgsPerGroup: CreateOrgData[],
+): Promise<{ existingOrgs: Org[]; newOrgs: CreateOrgData[] }> {
+  try {
+    return await filterOutExistingOrgs(requestManager, orgsPerGroup, groupId);
+  } catch (e) {
+    for (const org of orgsPerGroup) {
+      const { name } = org;
+      await logFailedOrg(
+        groupId,
+        name,
+        e.data
+          ? e.data.message
+          : e.message ||
+              'Failed to create org, please try again in DEBUG mode.',
+      );
+    }
+    throw new Error(
+      `All requested organizations failed to be created. Review the errors in ${path.resolve(
+        __dirname,
+        loggingPath,
+      )}/<groupId>.${FAILED_ORG_LOG_NAME}`,
+    );
+  }
+}

src/scripts/import-projects.ts

@@ -115,7 +115,7 @@ export async function importProjects(
 }> {
   const targetsFilePath = path.resolve(process.cwd(), loggingPath, fileName);
   if (!fs.existsSync(targetsFilePath)) {
-    throw new Error(`File not found ${targetsFilePath}`);
+    throw new Error(`File can not be found at location ${targetsFilePath}`);
   }
 
   let targets: ImportTarget[] = [];

test/lib/get-import-path.spec.ts

@@ -49,7 +49,7 @@ describe('getImportProjectsFile', () => {
     expect(() => {
       getImportProjectsFile('/fixtures/non-existent.json');
     }).toThrow(
-      "Please set the SNYK_IMPORT_PATH e.g. export SNYK_IMPORT_PATH='~/my/path/to/import-projects.json'",
+      "Please set the location via --file or SNYK_IMPORT_PATH e.g. export SNYK_IMPORT_PATH='~/my/path/to/import-projects.json",
     );
   });
 });

test/lib/orgs.test.ts

@@ -11,7 +11,6 @@ jest.requireActual('snyk-request-manager');
 
 describe('Orgs API', () => {
   const OLD_ENV = process.env;
-  process.env.GROUP_;
   process.env.SNYK_API = SNYK_API_TEST;
   process.env.SNYK_TOKEN = process.env.SNYK_TOKEN_TEST;
   const requestManager = new requestsManager({

test/lib/source-handlers/azure/azure.test.ts

@@ -17,7 +17,6 @@ beforeEach(() => {
     .reply(
       200,
       (uri: string) => {
-        console.log('URI = ' + uri);
         switch (uri) {
           case '/testOrg/_apis/projects?stateFilter=wellFormed&continuationToken=&api-version=4.1':
             return JSON.parse(
@@ -47,7 +46,6 @@ beforeEach(() => {
     .persist()
     .get(/.*/)
     .reply(200, (uri: string) => {
-      console.log('URI = ' + uri);
       switch (uri) {
         case '/reposTestOrg/_apis/projects?stateFilter=wellFormed&continuationToken=&api-version=4.1':
           return JSON.parse(

test/lib/source-handlers/github/github.test.ts

@@ -1,5 +1,4 @@
-import { listGithubOrgs } from '../../../../src/lib/source-handlers/github/list-organizations';
-import { listGithubRepos } from '../../../../src/lib/source-handlers/github/list-repos';
+import * as github from '../../../../src/lib/source-handlers/github';
 
 describe('listGithubOrgs script', () => {
   const OLD_ENV = process.env;
@@ -9,23 +8,23 @@ describe('listGithubOrgs script', () => {
   });
   it('list orgs', async () => {
     process.env.GITHUB_TOKEN = process.env.GH_TOKEN;
-    const orgs = await listGithubOrgs();
+    const orgs = await github.listGithubOrgs();
     expect(orgs[0]).toEqual({
       name: expect.any(String),
       id: expect.any(Number),
       url: expect.any(String),
     });
-  });
+  }, 10000);
   it('list orgs GHE', async () => {
     process.env.GITHUB_TOKEN = process.env.TEST_GHE_TOKEN;
     const GHE_URL = process.env.TEST_GHE_URL;
-    const orgs = await listGithubOrgs(GHE_URL);
+    const orgs = await github.listGithubOrgs(GHE_URL);
     expect(orgs[0]).toEqual({
       name: expect.any(String),
       id: expect.any(Number),
       url: expect.any(String),
     });
-  });
+  }, 10000);
 });
 
 describe('listGithubRepos script', () => {
@@ -38,14 +37,14 @@ describe('listGithubRepos script', () => {
     const GITHUB_ORG_NAME = process.env.TEST_GH_ORG_NAME;
     process.env.GITHUB_TOKEN = process.env.GH_TOKEN;
 
-    const orgs = await listGithubRepos(GITHUB_ORG_NAME as string);
+    const orgs = await github.listGithubRepos(GITHUB_ORG_NAME as string);
     expect(orgs[0]).toEqual({
       name: expect.any(String),
       owner: expect.any(String),
       branch: expect.any(String),
       fork: expect.any(Boolean),
     });
-  });
+  }, 10000);
   it('list GHE repos', async () => {
     const GITHUB_ORG_NAME = process.env.TEST_GH_ORG_NAME;
     const GHE_URL = process.env.TEST_GHE_URL;
@@ -61,5 +60,5 @@ describe('listGithubRepos script', () => {
       branch: expect.any(String),
       fork: expect.any(Boolean),
     });
-  });
+  }, 10000);
 });

test/scripts/create-orgs.test.ts

@@ -44,12 +44,11 @@ describe('createOrgs script', () => {
     );
     const logPath = path.resolve(__dirname + '/fixtures/create-orgs/1-org/');
     process.env.SNYK_LOG_PATH = logPath;
-    filesToDelete.push(path.resolve(logPath + `/abc.${CREATED_ORG_LOG_NAME}`));
+    filesToDelete.push(path.resolve(logPath + `/${GROUP_ID}.${CREATED_ORG_LOG_NAME}`));
 
     const { fileName, orgs, existing } = await createOrgs(importFile);
     createdOrgs.push(...orgs.map((o) => o.orgId));
     const log = path.resolve(logPath, fileName);
-
     filesToDelete.push(log);
     expect(orgs).not.toBeNull();
     expect(orgs[0]).toMatchObject({

test/scripts/generate-targets-data.test.ts

@@ -49,7 +49,7 @@ describe('generateTargetsImportDataFile Github script', () => {
       integrationId: 'github-********-********-********',
       orgId: 'org-id',
     });
-  });
+  }, 10000);
 
   it('generate Github Enterprise repo data', async () => {
     process.env.GITHUB_TOKEN = process.env.TEST_GHE_TOKEN;
@@ -87,7 +87,7 @@ describe('generateTargetsImportDataFile Github script', () => {
       integrationId: 'github-enterprise-********-********',
       orgId: 'org-id',
     });
-  });
+  }, 10000);
 
   it('generate Github repo data when no integrations are available', async () => {
     process.env.GITHUB_TOKEN = process.env.GH_TOKEN;
@@ -159,7 +159,7 @@ describe('generateTargetsImportDataFile Github script', () => {
       integrationId: expect.any(String),
       orgId: expect.any(String),
     });
-  });
+  }, 10000);
 });
 
 describe('generateTargetsImportDataFile Gitlab script', () => {

test/scripts/import-projects.test.ts

@@ -215,9 +215,12 @@ describe('Error handling', () => {
     const file = path.resolve(
       __dirname + '/fixtures/import-projects-invalid.json',
     );
-    expect(importProjects(file)).rejects.toThrow(
-      `Failed to parse targets from ${file}:\nUnexpected token } in JSON at position 120`,
-    );
+    expect(await importProjects(file)).toEqual({
+      filteredTargets: [],
+      projects: [],
+      skippedTargets: 0,
+      targets: [],
+    });
   }, 300);
 
   it('shows correct error when SNYK_LOG_PATH is not set', async () => {
@@ -259,7 +262,7 @@ describe('No projects scenarios', () => {
     );
     expect(projects.length === 0).toBeTruthy();
     // give file a little time to be finished to be written
-    await new Promise((r) => setTimeout(r, 1000));
+    await new Promise((r) => setTimeout(r, 3000));
     const logFile = fs.readFileSync(logFiles.importJobsLogPath, 'utf8');
     expect(logFile).toMatch(`"status":"complete","projects":[]}`);
     expect(logFile).toMatch(`"logs":[{"name":"snyk-fixtures/empty-repo"`);

test/system/__snapshots__/help.test.ts.snap

@@ -25,5 +25,6 @@ Commands:
 
 Options:
   --version  Show version number                                       [boolean]
-  --help     Show help                                                 [boolean]"
+  --help     Show help                                                 [boolean]
+  --file     Path to json file that contains the targets to be imported"
 `;