PXP-11183 feat dd filter bot on intake

package.json

@@ -22,6 +22,7 @@
     "brace": "^0.11.1",
     "clipboard-plus": "^1.1.0",
     "core-js": "^3.31.1",
+    "crawler-user-agents": "^1.0.115",
     "css-loader": "^3.4.2",
     "d3-array": "^1.2.0",
     "d3-axis": "^3.0.0",

src/ecosystemIndex.jsx

@@ -42,7 +42,7 @@ import ReduxQueryNode, { submitSearchForm } from './QueryNode/ReduxQueryNode';
 import {
   basename, gaTrackingId, workspaceUrl, workspaceErrorUrl, Error403Url,
   explorerPublic, enableResourceBrowser, resourceBrowserPublic, enableDAPTracker,
-  discoveryConfig, ddApplicationId, ddClientToken, ddEnv, ddUrl, ddSampleRate,
+  discoveryConfig, ddApplicationId, ddClientToken, ddEnv, ddUrl, ddSampleRate, ddKnownBotRegex,
 } from './localconf';
 import { portalVersion } from './versions';
 import Analysis from './Analysis/Analysis';
@@ -83,14 +83,15 @@ async function init() {
     // eslint-disable-next-line no-console
     console.warn('Datadog clientToken is set, but applicationId is missing');
   } else if (ddApplicationId && ddClientToken) {
+    const conditionalSampleRate = ddKnownBotRegex.test(navigator.userAgent) ? 0 : ddSampleRate;
     datadogRum.init({
       applicationId: ddApplicationId,
       clientToken: ddClientToken,
       site: ddUrl,
       service: 'portal',
       env: ddEnv,
       version: portalVersion,
-      sampleRate: ddSampleRate,
+      sampleRate: conditionalSampleRate,
       trackInteractions: true,
     });
   }

src/index.jsx

@@ -44,7 +44,7 @@ import ReduxQueryNode, { submitSearchForm } from './QueryNode/ReduxQueryNode';
 import {
   basename, gaTrackingId, workspaceUrl, workspaceErrorUrl, Error403Url,
   indexPublic, explorerPublic, enableResourceBrowser, resourceBrowserPublic, enableDAPTracker,
-  discoveryConfig, commonsWideAltText, ddApplicationId, ddClientToken, ddEnv, ddUrl, ddSampleRate,
+  discoveryConfig, commonsWideAltText, ddApplicationId, ddClientToken, ddEnv, ddUrl, ddSampleRate, ddKnownBotRegex,
 } from './localconf';
 import { portalVersion } from './versions';
 import Analysis from './Analysis/Analysis';
@@ -83,14 +83,15 @@ async function init() {
     // eslint-disable-next-line no-console
     console.warn('Datadog clientToken is set, but applicationId is missing');
   } else if (ddApplicationId && ddClientToken) {
+    const conditionalSampleRate = ddKnownBotRegex.test(navigator.userAgent) ? 0 : ddSampleRate;
     datadogRum.init({
       applicationId: ddApplicationId,
       clientToken: ddClientToken,
       site: ddUrl,
       service: 'portal',
       env: ddEnv,
       version: portalVersion,
-      sampleRate: ddSampleRate,
+      sampleRate: conditionalSampleRate,
       trackInteractions: true,
     });
   }

src/localconf.js

@@ -1,3 +1,5 @@
+import crawlers from 'crawler-user-agents';
+
 /* eslint-disable prefer-destructuring */
 const { components, requiredCerts, config } = require('./params');
 
@@ -160,6 +162,8 @@ function buildConfig(opts) {
       ddSampleRate = config.ddSampleRate;
     }
   }
+  const ddKnownBotPattern = crawlers.map((c) => c.pattern).join('|');
+  const ddKnownBotRegex = new RegExp(ddKnownBotPattern, 'i');
 
   // backward compatible: homepageChartNodes not set means using graphql query,
   // which will return 401 UNAUTHORIZED if not logged in, thus not making public
@@ -589,6 +593,7 @@ function buildConfig(opts) {
     ddEnv,
     ddUrl,
     ddSampleRate,
+    ddKnownBotRegex,
     showSystemUse,
     showSystemUseOnlyOnLogin,
     Error403Url,

src/localconf.test.js

@@ -19,9 +19,6 @@ describe('The localconf', () => {
 
   it('Defines a bunch of configuration variables', () => {
     expectedKeys.forEach((key) => {
-      // if (conf[key] === undefined) {
-      //   console.log(`conf[${key}] === ${conf[key]}`);
-      // }
       expect(conf[key]).toBeDefined();
     });
   });
@@ -38,4 +35,16 @@ describe('The localconf', () => {
       expect(newConf[key]).toEqual(conf[key]);
     });
   });
+
+  it('can identify bot by user-agent value', () => {
+    global.userAgent.mockReturnValue('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
+    const newConf = conf.buildConfig(conf.app);
+    expect(global.navigator.userAgent).toMatch(newConf.ddKnownBotRegex);
+  });
+
+  it('can ignore normal browser event by user-agent value', () => {
+    global.userAgent.mockReturnValue('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36');
+    const newConf = conf.buildConfig(conf.app);
+    expect(global.navigator.userAgent).not.toMatch(newConf.ddKnownBotRegex);
+  });
 });

src/nctIndex.jsx

@@ -57,7 +57,7 @@ import ReduxQueryNode, { submitSearchForm } from './QueryNode/ReduxQueryNode';
 import {
   basename, gaTrackingId, workspaceUrl, workspaceErrorUrl,
   indexPublic, explorerPublic, enableResourceBrowser, resourceBrowserPublic, enableDAPTracker,
-  discoveryConfig, ddApplicationId, ddClientToken, ddEnv, ddUrl, ddSampleRate,
+  discoveryConfig, ddApplicationId, ddClientToken, ddEnv, ddUrl, ddSampleRate, ddKnownBotRegex,
 } from './localconf';
 import { portalVersion } from './versions';
 import Analysis from './Analysis/Analysis';
@@ -96,14 +96,15 @@ async function init() {
     // eslint-disable-next-line no-console
     console.warn('Datadog clientToken is set, but applicationId is missing');
   } else if (ddApplicationId && ddClientToken) {
+    const conditionalSampleRate = ddKnownBotRegex.test(navigator.userAgent) ? 0 : ddSampleRate;
     datadogRum.init({
       applicationId: ddApplicationId,
       clientToken: ddClientToken,
       site: ddUrl,
       service: 'portal',
       env: ddEnv,
       version: portalVersion,
-      sampleRate: ddSampleRate,
+      sampleRate: conditionalSampleRate,
       trackInteractions: true,
     });
   }

src/setupJest.js

@@ -31,3 +31,5 @@ Object.defineProperty(window, 'matchMedia', {
     dispatchEvent: jest.fn(),
   })),
 });
+
+global.userAgent = jest.spyOn(navigator, 'userAgent', 'get');

src/workspaceIndex.jsx

@@ -28,7 +28,7 @@ import ReduxLogin, { fetchLogin } from './Login/ReduxLogin';
 import { ReduxNavBar, ReduxTopBar, ReduxFooter } from './Layout/reduxer';
 import {
   basename, gaTrackingId, workspaceUrl, workspaceErrorUrl, enableDAPTracker,
-  ddApplicationId, ddClientToken, ddEnv, ddUrl, ddSampleRate,
+  ddApplicationId, ddClientToken, ddEnv, ddUrl, ddSampleRate, ddKnownBotRegex,
 } from './localconf';
 import { portalVersion } from './versions';
 import { components } from './params';
@@ -58,14 +58,15 @@ async function init() {
     // eslint-disable-next-line no-console
     console.warn('Datadog clientToken is set, but applicationId is missing');
   } else if (ddApplicationId && ddClientToken) {
+    const conditionalSampleRate = ddKnownBotRegex.test(navigator.userAgent) ? 0 : ddSampleRate;
     datadogRum.init({
       applicationId: ddApplicationId,
       clientToken: ddClientToken,
       site: ddUrl,
       service: 'portal',
       env: ddEnv,
       version: portalVersion,
-      sampleRate: ddSampleRate,
+      sampleRate: conditionalSampleRate,
       trackInteractions: true,
     });
   }