From 612cabefbcb989dd8ee36a07e1a76248fbdc18a2 Mon Sep 17 00:00:00 2001 From: SergeyRyabinin Date: Wed, 15 Jan 2025 22:54:09 +0000 Subject: [PATCH] Refactor and simplify logic in sdksCommon cmake script to not rely on c2j model --- cmake/sdks.cmake | 1 - cmake/sdksCommon.cmake | 144 +++++++++++------------------------------ 2 files changed, 36 insertions(+), 109 deletions(-) diff --git a/cmake/sdks.cmake b/cmake/sdks.cmake index 9acf5386b0a..28c0db501f0 100644 --- a/cmake/sdks.cmake +++ b/cmake/sdks.cmake @@ -214,7 +214,6 @@ function(add_sdks) endif() add_subdirectory("${SDK_DIR}") - message(STATUS "exporting ${SDK_TARGET}") LIST(APPEND EXPORTS "${SDK_TARGET}") unset(SDK_TARGET) endforeach() diff --git a/cmake/sdksCommon.cmake b/cmake/sdksCommon.cmake index a4559aac578..3173a54a838 100644 --- a/cmake/sdksCommon.cmake +++ b/cmake/sdksCommon.cmake @@ -1,22 +1,11 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0. -# -# helper function that that gives primitive map functionality by treating a colon as the key-value separator, while the list semi-colon separates pairs -# to use, pass the list-map in as a third parameter (see helper functions below) -function(get_map_element KEY VALUE_VAR) - foreach(ELEMENT_PAIR ${ARGN}) - STRING(REGEX REPLACE "([^:]+):.*" "\\1" KEY_RESULT ${ELEMENT_PAIR}) - if(${KEY_RESULT} STREQUAL ${KEY} ) - STRING(REGEX REPLACE "[^:]+:(.*)" "\\1" VALUE_RESULT ${ELEMENT_PAIR}) - set(${VALUE_VAR} "${VALUE_RESULT}" PARENT_SCOPE) - return() - endif() - endforeach() - set(${VALUE_VAR} "" PARENT_SCOPE) -endfunction(get_map_element) - -function(get_multimap_element KEY VALUE_VAR) +# Return all values for a key from a flatmap +# I.e. given a flatmap list of items such as "key1:val1,val2,val3;key2:val2;key1:valDoNotLoseMe" and "key1" as arguments +# when called such as "get_flatmap_value(${TEST_NAME} TEMP_VAR ${TEST_DEPENDENCY_LIST})" +# will return "val1,val2,val3,valDoNotLoseMe" +function(get_flatmap_value KEY VALUE_VAR) set(${VALUE_VAR} "" PARENT_SCOPE) set(RETURN_LIST "") foreach(ELEMENT_PAIR ${ARGN}) @@ -27,39 +16,30 @@ function(get_multimap_element KEY VALUE_VAR) endif() endforeach() set(${VALUE_VAR} "${RETURN_LIST}" PARENT_SCOPE) -endfunction(get_multimap_element) - -# a bunch of key-value retrieval functions for the list-maps we defined above -function(get_c2j_date_for_service SERVICE_NAME C2J_DATE_VAR) - get_map_element(${SERVICE_NAME} TEMP_VAR ${C2J_LIST}) - set(${C2J_DATE_VAR} "${TEMP_VAR}" PARENT_SCOPE) -endfunction() - -function(get_c2j_name_for_service SERVICE_NAME C2J_NAME_VAR) - get_map_element(${SERVICE_NAME} TEMP_VAR ${C2J_SPECIAL_NAME_LIST}) - if(TEMP_VAR) - set(${C2J_NAME_VAR} "${TEMP_VAR}" PARENT_SCOPE) - else() - set(${C2J_NAME_VAR} "${SERVICE_NAME}" PARENT_SCOPE) - endif() -endfunction() +endfunction(get_flatmap_value) function(get_test_projects_for_service SERVICE_NAME TEST_PROJECT_NAME_VAR) - get_multimap_element(${SERVICE_NAME} TEMP_VAR ${SDK_TEST_PROJECT_LIST}) + get_flatmap_value(${SERVICE_NAME} TEMP_VAR ${SDK_TEST_PROJECT_LIST}) set(${TEST_PROJECT_NAME_VAR} "${TEMP_VAR}" PARENT_SCOPE) endfunction() function(get_dependencies_for_sdk PROJECT_NAME DEPENDENCY_LIST_VAR) - get_map_element(${PROJECT_NAME} TEMP_VAR ${SDK_DEPENDENCY_LIST}) + get_flatmap_value(${PROJECT_NAME} TEMP_VAR ${SDK_DEPENDENCY_LIST}) + if (TEMP_VAR) + string(REPLACE "," ";" TEMP_LIST_VAR ${TEMP_VAR}) + list(FIND TEMP_LIST_VAR "core" _index) + if (${_index} GREATER -1) # old cmake search in a list syntax + # core is already there + set(${DEPENDENCY_LIST_VAR} "${TEMP_VAR}" PARENT_SCOPE) + return() + endif() + endif() # "core" is the default dependency for every sdk. - # Since we removed the hand-written C2J_LIST and instead auto generating it based on models, - # and location of models may not exist or incorrect when SDK is installed and then source has been deleted by customers. - # we end up getting an incomplete C2J_LIST when customers call find_package(AWSSDK). But C2J_LIST is only used in customers code for dependencies completing. set(${DEPENDENCY_LIST_VAR} "${TEMP_VAR},core" PARENT_SCOPE) endfunction() function(get_dependencies_for_test TEST_NAME DEPENDENCY_LIST_VAR) - get_map_element(${TEST_NAME} TEMP_VAR ${TEST_DEPENDENCY_LIST}) + get_flatmap_value(${TEST_NAME} TEMP_VAR ${TEST_DEPENDENCY_LIST}) set(${DEPENDENCY_LIST_VAR} "${TEMP_VAR}" PARENT_SCOPE) endfunction() @@ -80,69 +60,21 @@ function(get_sdks_depending_on SDK_NAME DEPENDING_SDKS_VAR) set(${DEPENDING_SDKS_VAR} "${TEMP_SDK_LIST}" PARENT_SCOPE) endfunction() -# function that automatically picks up models from /code-generation/api-descriptions/ directory and build -# C2J_LIST needed for generation, services have multiple models will use the latest model (decided by model files' date) -# services have the name format abc.def.ghi will be renamed to ghi-def-abc (dot will not be accepted as Windows directory name ) -# and put into C2J_SPECIAL_NAME_LIST, but rumtime.lex will be renamed to lex based on historical reason. +# Function that reads /src/generated and builds a list of all available generated service clients for a build, +# The resulting $SERVICE_CLIENT_LIST is as simple as "cognito-identity,s3,dynamodb,etc,..." function(build_sdk_list) - file(GLOB ALL_MODEL_FILES "${CMAKE_CURRENT_SOURCE_DIR}/tools/code-generation/api-descriptions/*-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].normal.json") - foreach(model IN LISTS ALL_MODEL_FILES) - get_filename_component(modelName "${model}" NAME) - STRING(REGEX MATCH "([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])" date "${modelName}") - STRING(REGEX REPLACE "-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].normal.json" "" svc "${modelName}") - #special svc name conversion, e.g: runtime.lex->lex; abc.def.ghi->ghi-def-abc - if ("${svc}" STREQUAL "runtime.lex") - LIST(APPEND C2J_SPECIAL_NAME_LIST "lex:runtime.lex") - set(svc "lex") - elseif("${svc}" STREQUAL "runtime.lex.v2") - LIST(APPEND C2J_SPECIAL_NAME_LIST "lexv2-runtime:runtime.lex.v2") - set(svc "lexv2-runtime") - elseif("${svc}" STREQUAL "models.lex.v2") - LIST(APPEND C2J_SPECIAL_NAME_LIST "lexv2-models:models.lex.v2") - set(svc "lexv2-models") - elseif ("${svc}" STREQUAL "transfer") - LIST(APPEND C2J_SPECIAL_NAME_LIST "awstransfer:transfer") - set(svc "awstransfer") - elseif ("${svc}" STREQUAL "transcribe-streaming") - LIST(APPEND C2J_SPECIAL_NAME_LIST "transcribestreaming:transcribe-streaming") - set(svc "transcribestreaming") - elseif ("${svc}" STREQUAL "streams.dynamodb") - LIST(APPEND C2J_SPECIAL_NAME_LIST "dynamodbstreams:streams.dynamodb") - set (svc "dynamodbstreams") - elseif("${svc}" MATCHES "\\.") - string(REPLACE "." ";" nameParts ${svc}) - LIST(REVERSE nameParts) - string(REPLACE ";" "-" tmpSvc "${nameParts}") - LIST(APPEND C2J_SPECIAL_NAME_LIST "${tmpSvc}:${svc}") - set(svc "${tmpSvc}") - elseif("${svc}" STREQUAL "ec2") - if(PLATFORM_ANDROID AND CMAKE_HOST_WIN32) # ec2 isn't building for android on windows atm due to an internal compiler error, TODO: investigate further - continue() - endif() - elseif("${svc}" STREQUAL "s3") - LIST(APPEND C2J_SPECIAL_NAME_LIST "s3-crt:s3") - message(STATUS "Add s3-crt:s3 to C2J_SPECIAL_NAME_LIST") - endif() - get_map_element(${svc} existingDate ${C2J_LIST}) - if ("${existingDate}" STREQUAL "") - LIST(APPEND C2J_LIST "${svc}:${date}") - if ("${svc}" STREQUAL "s3") - LIST(APPEND C2J_LIST "s3-crt:${date}") - endif() - elseif(${existingDate} STRLESS ${date}) - LIST(REMOVE_ITEM C2J_LIST "${svc}:${existingDate}") - LIST(APPEND C2J_LIST "${svc}:${date}") - if ("${svc}" STREQUAL "s3") - LIST(REMOVE_ITEM C2J_LIST "s3-crt:${existingDate}") - LIST(APPEND C2J_LIST "s3-crt:${date}") - endif() - endif() + set(LIST_DIRECTORIES true) + file(GLOB ALL_GENERATED_CLIENTS $LIST_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/generated/src/aws-cpp-sdk-*") + + foreach(clientDir IN LISTS ALL_GENERATED_CLIENTS) + get_filename_component(clientDirName "${clientDir}" NAME) + STRING(REGEX REPLACE "^aws-cpp-sdk-" "" serviceName "${clientDirName}") + LIST(APPEND SERVICE_CLIENT_LIST "${serviceName}") endforeach() - set(C2J_LIST "${C2J_LIST}" PARENT_SCOPE) - set(C2J_SPECIAL_NAME_LIST "${C2J_SPECIAL_NAME_LIST}" PARENT_SCOPE) + set(SERVICE_CLIENT_LIST "${SERVICE_CLIENT_LIST}" PARENT_SCOPE) endfunction() - +# A list of so-called "hand-written SDK high-level libraries/components" set(HIGH_LEVEL_SDK_LIST "") list(APPEND HIGH_LEVEL_SDK_LIST "access-management") list(APPEND HIGH_LEVEL_SDK_LIST "identity-management") @@ -151,6 +83,7 @@ list(APPEND HIGH_LEVEL_SDK_LIST "transfer") list(APPEND HIGH_LEVEL_SDK_LIST "s3-encryption") list(APPEND HIGH_LEVEL_SDK_LIST "text-to-speech") +# A flatmap list of sdk_component:sdk_component_tests,sdk_component_another_tests set(SDK_TEST_PROJECT_LIST "") list(APPEND SDK_TEST_PROJECT_LIST "cloudfront:tests/aws-cpp-sdk-cloudfront-integration-tests") list(APPEND SDK_TEST_PROJECT_LIST "cognito-identity:tests/aws-cpp-sdk-cognitoidentity-integration-tests") @@ -184,11 +117,9 @@ build_sdk_list() if(EXISTS "${CMAKE_SOURCE_DIR}/generated") if(EXISTS "${CMAKE_SOURCE_DIR}/generated/tests") - foreach(GENERATED_C2J_TEST IN LISTS C2J_LIST) - STRING(REGEX REPLACE "([^:]+):.*" "\\1" GENERATED_C2J_TEST_RESULT ${GENERATED_C2J_TEST}) - - if(EXISTS "${CMAKE_SOURCE_DIR}/generated/tests/${GENERATED_C2J_TEST_RESULT}-gen-tests") - list(APPEND SDK_TEST_PROJECT_LIST "${GENERATED_C2J_TEST_RESULT}:generated/tests/${GENERATED_C2J_TEST_RESULT}-gen-tests") + foreach(SERVICE_NAME IN LISTS SERVICE_CLIENT_LIST) + if(EXISTS "${CMAKE_SOURCE_DIR}/generated/tests/${SERVICE_NAME}-gen-tests") + list(APPEND SDK_TEST_PROJECT_LIST "${SERVICE_NAME}:generated/tests/${SERVICE_NAME}-gen-tests") endif() endforeach() endif() @@ -213,10 +144,7 @@ list(APPEND TEST_DEPENDENCY_LIST "text-to-speech:polly,core") list(APPEND TEST_DEPENDENCY_LIST "transfer:s3,core") list(APPEND TEST_DEPENDENCY_LIST "logs:access-management,cognito-identity,iam,core") -# make a list of the generated clients -set(GENERATED_SERVICE_LIST "") -foreach(GENERATED_C2J_SERVICE IN LISTS C2J_LIST) - STRING(REGEX REPLACE "([^:]+):.*" "\\1" SERVICE_RESULT ${GENERATED_C2J_SERVICE}) - list(APPEND GENERATED_SERVICE_LIST ${SERVICE_RESULT}) - list(APPEND SDK_DEPENDENCY_LIST "${SERVICE_RESULT}:core") +set(GENERATED_SERVICE_LIST ${SERVICE_CLIENT_LIST}) +foreach(SERVICE_NAME IN LISTS SERVICE_CLIENT_LIST) + list(APPEND SDK_DEPENDENCY_LIST "${SERVICE_NAME}:core") endforeach()