-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: [MC-681] Implement scheduled_corpus_candidate Snowplow functions
chore: move common Snowplow logic to content-common chore: use shared Snowplow code in prospect-api and curated-corpus-api fix: Snowplow retries on error chore: move Snowplow Micro to integration test file fix: lint error snowplowHttpProtocol unused feat: emit Snowplow events on error remove duplicate test coverage for image error fix snowplow tests fix ECONNREFUSED to Snowplow Micro in CI tests Revert "fix ECONNREFUSED to Snowplow Micro in CI tests" This reverts commit 470df13. fix: move Snowplow Micro tests to integrations add test for Snowplow event on success
- Loading branch information
Showing
30 changed files
with
839 additions
and
404 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
56 changes: 56 additions & 0 deletions
56
lambdas/corpus-scheduler-lambda/src/events/snowplow.integration.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
import { | ||
resetSnowplowEvents, | ||
waitForSnowplowEvents, | ||
} from 'content-common/events/snowplow/test-helpers'; | ||
import { getEmitter, getTracker } from 'content-common/events/snowplow'; | ||
import { queueSnowplowEvent } from './snowplow'; | ||
import { | ||
SnowplowScheduledCorpusCandidateErrorName, | ||
SnowplowScheduledCorpusCandidate, | ||
} from './types'; | ||
import { random } from 'typia'; | ||
import config from '../config'; | ||
|
||
describe('snowplow', () => { | ||
const mockCandidate = { | ||
...random<SnowplowScheduledCorpusCandidate>(), | ||
scheduled_corpus_item_external_id: '05706565-5a9c-4b57-83e5-a426485a4714', | ||
approved_corpus_item_external_id: 'c43a2aa5-28de-4828-a20e-1fdf60cc4a80', | ||
}; | ||
const emitter = getEmitter(); | ||
const tracker = getTracker(emitter, config.snowplow.appId); | ||
|
||
beforeEach(async () => { | ||
await resetSnowplowEvents(); | ||
}); | ||
|
||
it('should accept an event with a scheduled corpus candidate', async () => { | ||
queueSnowplowEvent(tracker, mockCandidate); | ||
|
||
const allEvents = await waitForSnowplowEvents(); | ||
|
||
expect(allEvents.total).toEqual(1); | ||
expect(allEvents.bad).toEqual(0); | ||
}); | ||
|
||
describe('error events', () => { | ||
Object.values(SnowplowScheduledCorpusCandidateErrorName).forEach( | ||
(errorName) => { | ||
it(`should emit events with ${errorName} error`, async () => { | ||
queueSnowplowEvent(tracker, { | ||
...mockCandidate, | ||
scheduled_corpus_item_external_id: undefined, | ||
approved_corpus_item_external_id: undefined, | ||
error_name: errorName, | ||
error_description: `Oh no! A ${errorName} error occurred.`, | ||
}); | ||
|
||
const allEvents = await waitForSnowplowEvents(); | ||
|
||
expect(allEvents.total).toEqual(1); | ||
expect(allEvents.bad).toEqual(0); | ||
}); | ||
}, | ||
); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
import { | ||
buildSelfDescribingEvent, | ||
Tracker, | ||
SelfDescribingEvent, | ||
SelfDescribingJson, | ||
} from '@snowplow/node-tracker'; | ||
|
||
import config from '../config'; | ||
import { | ||
SnowplowScheduledCorpusCandidateErrorName, | ||
SnowplowScheduledCorpusCandidate, | ||
} from './types'; | ||
import { ScheduledCandidate } from '../types'; | ||
|
||
/** | ||
* creates an object_update Snowplow event for scheduled_corpus_candidate | ||
* | ||
* @returns SelfDescribingEvent | ||
*/ | ||
export const generateEvent = (): SelfDescribingEvent => { | ||
return { | ||
event: { | ||
schema: config.snowplow.schemas.objectUpdate, | ||
data: { | ||
trigger: 'scheduled_corpus_candidate_generated', | ||
object: 'scheduled_corpus_candidate', | ||
}, | ||
}, | ||
}; | ||
}; | ||
|
||
/** | ||
* generates a scheduled_corpus_candidate entity being sent to snowplow | ||
* | ||
* @param scheduledCorpusCandidate PocketAnalyticsArticle | ||
* @returns SelfDescribingJson | ||
*/ | ||
export const generateContext = ( | ||
scheduledCorpusCandidate: SnowplowScheduledCorpusCandidate, | ||
): SelfDescribingJson => { | ||
return { | ||
schema: config.snowplow.schemas.scheduled_corpus_candidate, | ||
data: scheduledCorpusCandidate, | ||
}; | ||
}; | ||
|
||
/** | ||
* | ||
* @param candidate ML candidate | ||
* @param errorName Snowplow structured error | ||
* @param errorDescription Longer human-readable description of the error | ||
*/ | ||
export const generateSnowplowErrorEntity = ( | ||
candidate: ScheduledCandidate, | ||
errorName: SnowplowScheduledCorpusCandidateErrorName, | ||
errorDescription: string, | ||
): SnowplowScheduledCorpusCandidate => { | ||
return { | ||
scheduled_corpus_candidate_id: candidate.scheduled_corpus_candidate_id, | ||
candidate_url: candidate.scheduled_corpus_item.url, | ||
features: candidate.features, | ||
run_details: candidate.run_details, | ||
error_name: errorName, | ||
error_description: errorDescription, | ||
}; | ||
}; | ||
|
||
/** | ||
* | ||
* @param candidate ML candidate | ||
* @param approvedCorpusItemId Identifier the item added to the corpus | ||
*/ | ||
export const generateSnowplowSuccessEntity = ( | ||
candidate: ScheduledCandidate, | ||
approvedCorpusItemId: string, | ||
): SnowplowScheduledCorpusCandidate => { | ||
return { | ||
scheduled_corpus_candidate_id: candidate.scheduled_corpus_candidate_id, | ||
candidate_url: candidate.scheduled_corpus_item.url, | ||
approved_corpus_item_external_id: approvedCorpusItemId, | ||
features: candidate.features, | ||
run_details: candidate.run_details, | ||
// TODO: set scheduled_corpus_item_external_id | ||
}; | ||
}; | ||
|
||
/** | ||
* main entry point to snowplow. queues up an event to send. | ||
* | ||
* (elsewhere, we tell snowplow to send all queued events.) | ||
* | ||
* @param tracker TrackerInterface | ||
* @param entity Entity representing the result of trying to schedule a candidate. | ||
*/ | ||
export const queueSnowplowEvent = ( | ||
tracker: Tracker, | ||
entity: SnowplowScheduledCorpusCandidate, | ||
) => { | ||
const event = generateEvent(); | ||
const contexts: SelfDescribingJson[] = [generateContext(entity)]; | ||
|
||
// reminder - this method is not async and does not directly initiate | ||
// any http request. it sends the event to a queue internal to the | ||
// snowplow module, which has its own logic on when to flush the queue. | ||
tracker.track(buildSelfDescribingEvent(event), contexts); | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
import { SnowplowScheduledCorpusCandidate } from './types'; | ||
import { | ||
getGoodSnowplowEvents, | ||
parseSnowplowData, | ||
} from 'content-common/events/snowplow/test-helpers'; | ||
|
||
/** | ||
* @return scheduled_corpus_candidate entity from the last good event sent to Snowplow Micro. | ||
*/ | ||
export async function extractScheduledCandidateEntity(): Promise<SnowplowScheduledCorpusCandidate> { | ||
const goodEvents = await getGoodSnowplowEvents(); | ||
const snowplowContext = parseSnowplowData( | ||
goodEvents[0].rawEvent.parameters.cx, | ||
); | ||
return snowplowContext.data[0].data as SnowplowScheduledCorpusCandidate; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
import { | ||
ScheduledCorpusCandidateFeatures, | ||
ScheduledCorpusCandidateRunDetails, | ||
} from '../types'; | ||
|
||
// scheduled_corpus_candidate entity | ||
export type SnowplowScheduledCorpusCandidate = { | ||
scheduled_corpus_candidate_id: string; | ||
scheduled_corpus_item_external_id?: string; | ||
approved_corpus_item_external_id?: string; | ||
candidate_url: string; | ||
error_name?: SnowplowScheduledCorpusCandidateErrorName; | ||
error_description?: string; | ||
features: ScheduledCorpusCandidateFeatures; | ||
run_details: ScheduledCorpusCandidateRunDetails; | ||
}; | ||
|
||
export enum SnowplowScheduledCorpusCandidateErrorName { | ||
ALREADY_SCHEDULED = 'ALREADY_SCHEDULED', | ||
/** TODO: [MC-737] Add validation on scheduled date. */ | ||
INSUFFICIENT_TIME_BEFORE_SCHEDULED_DATE = 'INSUFFICIENT_TIME_BEFORE_SCHEDULED_DATE', | ||
/** TODO: [MC-666] Add safeguard to prevent scheduling from unverified domains. */ | ||
DOMAIN_NOT_ALLOWED_FOR_AUTO_SCHEDULING = 'DOMAIN_NOT_ALLOWED_FOR_AUTO_SCHEDULING', | ||
MISSING_EXCERPT = 'MISSING_EXCERPT', | ||
MISSING_TITLE = 'MISSING_TITLE', | ||
MISSING_IMAGE = 'MISSING_IMAGE', | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.