Merge pull request #7 from aws-solutions/release/v2.0.3

Stage files for v2.0.3 release

CHANGELOG.md

@@ -1,17 +1,29 @@
 # Change Log
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.0.3] - 2024-02-20
+
+### Added
+
+- Fix user's ability to opt out anonymized data collection.
+- Fix SageMaker notebook instance lifecycle configuration to auto-stop the compute instance if it’s idle for 900 seconds.
+
 ## [2.0.2] - 2024-01-03
+
 ### Added
+
 - Added timestamps to processed file names for prevention of accidental overwrites
 - Fixed null values casting as -1
 - Updated state machine to only trigger on successful file uploads
 
 ## [2.0.1] - 2023-10-25
+
 ### Added
+
 - Update urllib to v1.26.18
 - Fix operational policy permissions
 
@@ -33,16 +45,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add functional and unit tests.
 - Add AppRegistry support.
 - Improve codebase to meet Solution quality bar.
-- Reforge and simplify six stacks to one stack. 
-- Allow multiple stack deployment in one region. 
+- Reforge and simplify six stacks to one stack.
+- Allow multiple stack deployment in one region.
 - Update folder structure and add files for Solutions layout.
 - Incorporate CDK solution helper and update build output.
 - Update runtimes, layers, timeouts, architectures, dependencies, package hierarchy, and copyrights.
 - Reforge AWS Data Wrangler layer build.
 - Update DynamoDB tables to on-demand capacity.
 - Update scripts for uninstalling solution.
 - Add, update and fix IAM role and policy.
-- Restrict KMS permissions. 
+- Restrict KMS permissions.
 - Bug fix and enhancements
 
 ## [1.1.3] - 2023-06-01

README.md

@@ -71,7 +71,7 @@ To customize the solution, follow the steps below:
 The following procedures assumes that all the OS-level configuration has been completed. They are:
 
 * [AWS Command Line Interface](https://aws.amazon.com/cli/)
-* [Python](https://www.python.org/) 3.9 or newer
+* [Python](https://www.python.org/) 3.11 or newer
 * [Node.js](https://nodejs.org/en/) 16.x or newer
 * [AWS CDK](https://aws.amazon.com/cdk/) 2.60.0 or newer
 

deployment/run-unit-tests.sh

@@ -40,7 +40,7 @@ if [ $? == 1 ]; then
     if [ -d $venv_folder ]; then
       rm -rf $venv_folder
     fi
-    python3 -m venv $venv_folder
+    python3.11 -m venv $venv_folder
     source $venv_folder/bin/activate
 
     using_test_venv=1
@@ -53,6 +53,12 @@ else
   echo "------------------------------------------------------------------------------"
   echo "[Env] Using active virtual environment for tests"
   echo "------------------------------------------------------------------------------"
+  python_version=$(python --version 2>&1 | cut -d ' ' -f 2)
+  if [[ "$python_version" != "11"* ]]; then
+    echo "You are using Python version $python_version. Python version 11 is required."
+    echo "Update your environment or run tests again without an active environment."
+    exit 1
+  fi
   echo ''
 fi
 

solution-manifest.yaml

@@ -1,8 +1,8 @@
 id: SO0193
 name: amazon-marketing-cloud-insights-on-aws
-version: v2.0.2
+version: v2.0.3
 cloudformation_templates:
   - template: amazon-marketing-cloud-insights.template
     main_template: true
 build_environment:
-  build_image: 'aws/codebuild/standard:6.0'
+  build_image: 'aws/codebuild/standard:7.0'

source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/aws_lambda/cfn_custom_resources/solutions_metrics/metrics.py

@@ -19,7 +19,7 @@
 from cdk_nag import NagSuppressions
 
 class Metrics(Construct):
-    """Used to track anonymous solution deployment metrics."""
+    """Used to track anonymized solution deployment metrics."""
 
     def __init__(
         self,
@@ -55,14 +55,14 @@ def __init__(
             ],
         )
 
-        self._send_anonymous_usage_data = CfnCondition(
+        self._send_anonymized_usage_data = CfnCondition(
             self,
-            "SendAnonymousUsageData",
+            "SendAnonymizedData",
             expression=Fn.condition_equals(
-                Fn.find_in_map("Solution", "Data", "SendAnonymousUsageData"), "Yes"
+                Fn.find_in_map("Solution", "Data", "SendAnonymizedData"), "Yes"
             ),
         )
-        self._send_anonymous_usage_data.override_logical_id("SendAnonymousUsageData")
+        self._send_anonymized_usage_data.override_logical_id("SendAnonymizedData")
 
         properties = {
             "ServiceToken": self._metrics_function.function_arn,
@@ -73,12 +73,12 @@ def __init__(
         }
         self.solution_metrics = CfnResource(
             self,
-            "SolutionMetricsAnonymousData",
-            type="Custom::AnonymousData",
+            "SolutionMetricsAnonymizedData",
+            type="Custom::AnonymizedData",
             properties=properties,
         )
-        self.solution_metrics.override_logical_id("SolutionMetricsAnonymousData")
-        self.solution_metrics.cfn_options.condition = self._send_anonymous_usage_data
+        self.solution_metrics.override_logical_id("SolutionMetricsAnonymizedData")
+        self.solution_metrics.cfn_options.condition = self._send_anonymized_usage_data
 
         NagSuppressions.add_resource_suppressions(
             self._metrics_function.role,

source/cdk_solution_helper_py/helpers_cdk/aws_solutions/cdk/mappings.py

@@ -10,21 +10,21 @@ def __init__(
         self,
         parent: Construct,
         solution_id: str,
-        send_anonymous_usage_data: bool = True,
+        send_anonymized_usage_data: bool = True,
         quicksight_template_arn: bool = False,
     ):
         self.parent = parent
 
-        # Track the solution mapping (ID, version, anonymous usage data)
+        # Track the solution mapping (ID, version, anonymized usage data)
         self.solution_mapping = CfnMapping(
             parent,
             "Solution",
             mapping={
                 "Data": {
                     "ID": solution_id,
                     "Version": "%%SOLUTION_VERSION%%",
-                    "SendAnonymousUsageData": "Yes"
-                    if send_anonymous_usage_data
+                    "SendAnonymizedData": "Yes"
+                    if send_anonymized_usage_data
                     else "No",
                 }
             }

source/infrastructure/amc_insights/amc_insights_stack.py

@@ -12,7 +12,7 @@
 from data_lake.datasets import SDLFDatasetConstruct
 from amc_insights.microservices.platform_management_service import PlatformManagerSageMaker
 from amc_insights.microservices.tenant_provisioning_service import TenantProvisioningService
-from amc_insights.custom_resource.anonymous_operational_metrics import OperationalMetrics
+from amc_insights.custom_resource.anonymized_operational_metrics import OperationalMetrics
 from amc_insights.custom_resource.cloudwatch_metrics.cloudwatch_metrics import CloudwatchMetrics
 from amc_insights.microservices.workflow_manager_service import WorkFlowManagerService
 from aws_cdk import CfnParameter, CfnCondition, Fn, Aspects

source/infrastructure/amc_insights/custom_resource/anonymous_operational_metrics/lambdas/stack_uuid.py → source/infrastructure/amc_insights/custom_resource/anonymized_operational_metrics/lambdas/stack_uuid.py

@@ -13,7 +13,7 @@
 helper = CfnResource()
 
 STACK_NAME = os.environ['STACK_NAME']
-secret_name = f"{STACK_NAME}-anonymous-metrics-uuid"
+secret_name = f"{STACK_NAME}-anonymized-metrics-uuid"
 
 
 def event_handler(event, context):
@@ -35,7 +35,7 @@ def on_create(event, _):
 
 def create_uuid():
     """
-    This function is responsible for creating the Secrets Manager uuid for anonymous metrics.
+    This function is responsible for creating the Secrets Manager uuid for anonymized metrics.
     """
 
     secrets_manager_client = get_service_client("secretsmanager")

source/infrastructure/amc_insights/custom_resource/anonymous_operational_metrics/operational_metrics.py → source/infrastructure/amc_insights/custom_resource/anonymized_operational_metrics/operational_metrics.py

@@ -45,7 +45,7 @@ def _create_iam_policy_for_custom_resource_lambda(self):
                 "secretsmanager:GetSecretValue"
             ],
             resources=[
-                f"arn:aws:secretsmanager:{Aws.REGION}:{Aws.ACCOUNT_ID}:secret:{Aws.STACK_NAME}-anonymous-metrics-uuid*"],
+                f"arn:aws:secretsmanager:{Aws.REGION}:{Aws.ACCOUNT_ID}:secret:{Aws.STACK_NAME}-anonymized-metrics-uuid*"],
         )
 
         self.operational_metrics_lambda_iam_policy = iam.Policy(
@@ -57,15 +57,15 @@ def _create_iam_policy_for_custom_resource_lambda(self):
 
     def _create_operational_metrics_lambda(self):
         """
-        This function is responsible for creating the anonymous operational metrics uuid in Secrets Manager.
+        This function is responsible for creating the anonymized operational metrics uuid in Secrets Manager.
         """
         self._operational_metrics_lambda = SolutionsPythonFunction(
             self,
             "CreateOperationalMetrics",
-            AMC_INSIGHTS_CUSTOM_RESOURCE_PATH / "anonymous_operational_metrics" / "lambdas" / "stack_uuid.py",
+            AMC_INSIGHTS_CUSTOM_RESOURCE_PATH / "anonymized_operational_metrics" / "lambdas" / "stack_uuid.py",
             "event_handler",
             runtime=lambda_.Runtime.PYTHON_3_9,
-            description="Lambda function for custom resource for the creating anonymous operational metrics uuid in Secrets Manager",
+            description="Lambda function for custom resource for the creating anonymized operational metrics uuid in Secrets Manager",
             timeout=Duration.minutes(5),
             memory_size=256,
             architecture=lambda_.Architecture.ARM_64,
@@ -91,7 +91,7 @@ def _create_operational_metrics_lambda(self):
 
     def _create_operational_metrics_custom_resource(self):
         """
-        This function creates the customer resource for creating the anonymous operational metrics uuid in Secrets Manager.
+        This function creates the customer resource for creating the anonymized operational metrics uuid in Secrets Manager.
         """
         self._operational_metrics_custom_resource = CustomResource(
             self,

source/infrastructure/amc_insights/custom_resource/cloudwatch_metrics/cloudwatch_metrics.py

@@ -5,6 +5,7 @@
 from aws_cdk import (
     Duration,
     Aws,
+    Fn,
     aws_lambda as lambda_,
     aws_iam as iam
 )
@@ -33,14 +34,18 @@ def __init__(
         self._create_cloudwatch_metrics_function()
         self._create_event_bridge_rule()
 
+    @staticmethod
+    def send_anonymized_data():
+        return Fn.find_in_map("Solution", "Data", "SendAnonymizedData")
+
     def _create_iam_policy(self):
         secrets_manager_statement = PolicyStatement(
             effect=Effect.ALLOW,
             actions=[
                 "secretsmanager:GetSecretValue"
             ],
             resources=[
-                f"arn:aws:secretsmanager:{Aws.REGION}:{Aws.ACCOUNT_ID}:secret:{Aws.STACK_NAME}-anonymous-metrics-uuid*"],
+                f"arn:aws:secretsmanager:{Aws.REGION}:{Aws.ACCOUNT_ID}:secret:{Aws.STACK_NAME}-anonymized-metrics-uuid*"],
         )
         cloudwatch_statement = PolicyStatement(
             effect=Effect.ALLOW,
@@ -85,7 +90,8 @@ def _create_cloudwatch_metrics_function(self):
                 "STACK_NAME": Aws.STACK_NAME,
                 "SOLUTION_ID": self.node.try_get_context("SOLUTION_ID"),
                 "SOLUTION_VERSION": self.node.try_get_context("SOLUTION_VERSION"),
-                "METRICS_NAMESPACE": self.node.try_get_context("METRICS_NAMESPACE")
+                "METRICS_NAMESPACE": self.node.try_get_context("METRICS_NAMESPACE"),
+                "SEND_ANONYMIZED_DATA": self.send_anonymized_data(),
             },
             layers=[
                 SolutionsLayer.get_or_create(self)

source/infrastructure/amc_insights/custom_resource/cloudwatch_metrics/lambdas/report.py

@@ -17,6 +17,7 @@
 SOLUTION_ID = os.environ["SOLUTION_ID"]
 SOLUTION_VERSION = os.environ["SOLUTION_VERSION"]
 METRICS_NAMESPACE = os.environ["METRICS_NAMESPACE"]
+SEND_ANONYMIZED_DATA = os.environ["SEND_ANONYMIZED_DATA"]
 
 SECONDS_IN_A_DAY = 86400
 
@@ -44,7 +45,11 @@ def event_handler(event, context):
     logger.info("We got the following event:\n")
     logger.info("event:\n {s}".format(s=event))
     logger.info("context:\n {s}".format(s=context))
-    send_metrics()
+    if SEND_ANONYMIZED_DATA == "Yes":
+        logger.info("Report anonymized operational metrics")
+        send_metrics()
+    else:
+        logger.info("Anonymized data collection is opted out, no operational metrics to report")
 
 
 def send_metrics():
@@ -57,7 +62,7 @@ def send_metrics():
     end_time = datetime.utcnow()
     start_time = (end_time - timedelta(seconds=SECONDS_IN_A_DAY))
     uuid = secrets_manager_client.get_secret_value(
-        SecretId=f"{os.environ['STACK_NAME']}-anonymous-metrics-uuid"
+        SecretId=f"{os.environ['STACK_NAME']}-anonymized-metrics-uuid"
     )['SecretString']
     # Assemble the payload structure for reporting:
     data = {
@@ -119,7 +124,8 @@ def send_metrics():
         # Add datapoints to the reporting payload:
         if datapoints:
             if len(datapoints) > 1:
-                logging.warning("Got " + str(len(datapoints)) + " datapoints but only expected one datapoint since period is one day and start/end time spans one day.")
+                logging.warning("Got " + str(
+                    len(datapoints)) + " datapoints but only expected one datapoint since period is one day and start/end time spans one day.")
             total = 0
             for datapoint in datapoints:
                 # There should only be one datapoint since period is one day and
@@ -135,4 +141,4 @@ def send_metrics():
         response = requests.post(METRICS_ENDPOINT, json=data, timeout=5)
         print(f"Response status code = {response.status_code}")
     else:
-        logging.info("No data to report.")
+        logging.info("No data to report.")

source/infrastructure/amc_insights/microservices/platform_management_service/platform_management_sagemaker.py

@@ -9,7 +9,6 @@
     Aspects,
     CfnCondition,
     CfnOutput,
-    aws_lakeformation as lakeformation,
     aws_kms as kms,
     aws_sagemaker as sagemaker,
 )
@@ -46,6 +45,7 @@ def __init__(
         self._creating_resources_condition = creating_resources_condition
 
         self._notebook_samples_prefix = "platform_notebook_manager_samples"
+        self._notebook_instance_name = f"{self._resource_prefix}-amc-insights-platform-manager-notebooks"
 
         PlatformManagerUploader(
             self, "SyncPlatformManager",
@@ -86,6 +86,17 @@ def _create_sagemaker_role(self) -> None:
             roles=[self.sagemaker_role],
             document=PolicyDocument(
                 statements=[
+                    # give sagemaker permission to execute Lifecycle configuration script to stop the instance if it's idle
+                    PolicyStatement(
+                        effect=Effect.ALLOW,
+                        actions=["sagemaker:DescribeNotebookInstance",
+                                 "sagemaker:StopNotebookInstance"
+                                 ],
+                        resources=[
+                            f"arn:aws:sagemaker:{Aws.REGION}:{Aws.ACCOUNT_ID}:notebook-instance/{self._notebook_instance_name}",
+                        ]
+                    ),
+
                     # give sagemaker permission to invoke the WFM SM startup lambdas
                     PolicyStatement(
                         effect=Effect.ALLOW,
@@ -149,7 +160,6 @@ def _create_sagemaker_role(self) -> None:
             )
         )
 
-
     def _create_sagemaker_lifecycle_config(self) -> None:
         """lifecycle config and notebook instance"""
 
@@ -165,7 +175,6 @@ def _create_sagemaker_lifecycle_config(self) -> None:
                     set -e
 
                     # Parameters
-                    IDLE_TIME=900
                     S3_BUCKET={self._solution_buckets.artifacts_bucket.bucket_name}
                     INVOKE_WORKFLOW_EXECUTION_SM_NAME="INVOKE_WORKFLOW_EXECUTION_SM_NAME={self._workflow_manager_resources.lambda_invoke_workflow_execution_sm.function_name}"
                     INVOKE_WORKFLOW_SM_NAME="INVOKE_WORKFLOW_SM_NAME={self._workflow_manager_resources.lambda_invoke_workflow_sm.function_name}"
@@ -202,7 +211,7 @@ def _create_sagemaker_lifecycle_config(self) -> None:
                     echo "Fetching the autostop script"
                     wget https://raw.githubusercontent.com/aws-samples/amazon-sagemaker-notebook-instance-lifecycle-config-samples/master/scripts/auto-stop-idle/autostop.py
                     echo "Starting the SageMaker autostop script in cron"
-                    (crontab -l 2>/dev/null; echo "*/5 * * * * /usr/bin/python $PWD/autostop.py --time $IDLE_TIME --ignore-connections") | crontab -
+                    (crontab -l 2>/dev/null; echo "*/5 * * * * $(which python) $PWD/autostop.py --time 900 --ignore-connections >> /var/log/autostop.log 2>&1") | crontab -
 
                     # Remove lost+found folder
                     rm -rf /home/ec2-user/SageMaker/lost+found
@@ -220,7 +229,7 @@ def _create_sagemaker_notebook_instance(self):
             role_arn=self.sagemaker_role.role_arn,
             kms_key_id=self._sagemaker_kms_key.key_id,
             lifecycle_config_name=self.sagemaker_lifecycle_config.attr_notebook_instance_lifecycle_config_name,
-            notebook_instance_name=f"{self._resource_prefix}-amc-insights-platform-manager-notebooks",
+            notebook_instance_name=self._notebook_instance_name,
             root_access="Enabled",
         )
 

source/infrastructure/amc_insights/microservices/tenant_provisioning_service/lambdas/InvokeTPSInitializeSM/handler.py

@@ -81,7 +81,7 @@ def handler(event, _):
     message = f"created state machine execution response : {response} for customer {customer_details_formatted['TenantName']}"
     logger.info(message)
 
-    # Record anonymous metric
+    # Record anonymized metric
     metrics.Metrics(METRICS_NAMESPACE, STACK_NAME, logger).put_metrics_count_value_1(metric_name="InvokeTPSInitializeSM")
 
     return json.dumps(response, default=json_encoder_default)