diff --git a/lisa/sut_orchestrator/azure/arm_template.bicep b/lisa/sut_orchestrator/azure/arm_template.bicep index d067f9ed3b..c213f5ad44 100644 --- a/lisa/sut_orchestrator/azure/arm_template.bicep +++ b/lisa/sut_orchestrator/azure/arm_template.bicep @@ -98,7 +98,7 @@ func getEphemeralOSImage(node object) object => { name: '${node.name}-osDisk' diffDiskSettings: { option: 'local' - placement: 'CacheDisk' + placement: node.disk.ephemeral_disk_placement_type } caching: 'ReadOnly' createOption: 'FromImage' @@ -333,7 +333,7 @@ resource nodes_data_disks 'Microsoft.Compute/disks@2022-03-02' = [ } ] -resource nodes_vms 'Microsoft.Compute/virtualMachines@2022-08-01' = [for i in range(0, node_count): { +resource nodes_vms 'Microsoft.Compute/virtualMachines@2024-03-01' = [for i in range(0, node_count): { name: nodes[i].name location: nodes[i].location tags: combined_vm_tags diff --git a/lisa/sut_orchestrator/azure/arm_template.json b/lisa/sut_orchestrator/azure/arm_template.json new file mode 100644 index 0000000000..789d863560 --- /dev/null +++ b/lisa/sut_orchestrator/azure/arm_template.json @@ -0,0 +1,881 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "17292510441982444978" + } + }, + "functions": [ + { + "namespace": "__bicep", + "members": { + "isCvm": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "bool", + "value": "[bool(and(not(empty(parameters('node').vhd)), not(empty(parameters('node').vhd.vmgs_path))))]" + } + }, + "isVhd": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "bool", + "value": "[bool(and(not(empty(parameters('node').vhd)), not(empty(parameters('node').vhd.vhd_path))))]" + } + }, + "getOSDisk": { + "parameters": [ + { + "type": "string", + "name": "diskName" + } + ], + "output": { + "type": "object", + "value": { + "createOption": "Attach", + "osType": "Linux", + "managedDisk": { + "id": "[resourceId('Microsoft.Compute/disks', parameters('diskName'))]" + } + } + } + }, + "getOsDiskVhd": { + "parameters": [ + { + "type": "string", + "name": "vmName" + } + ], + "output": { + "type": "object", + "value": { + "id": "[resourceId('Microsoft.Compute/images', format('{0}-image', parameters('vmName')))]" + } + } + }, + "getLinuxConfiguration": { + "parameters": [ + { + "type": "string", + "name": "keyPath" + }, + { + "type": "string", + "name": "publicKeyData" + }, + { + "type": "bool", + "name": "disable_password_authentication" + } + ], + "output": { + "type": "object", + "value": { + "disablePasswordAuthentication": "[parameters('disable_password_authentication')]", + "ssh": { + "publicKeys": [ + { + "path": "[parameters('keyPath')]", + "keyData": "[parameters('publicKeyData')]" + } + ] + }, + "provisionVMAgent": true + } + } + }, + "getEphemeralOSImage": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "object", + "value": { + "name": "[format('{0}-osDisk', parameters('node').name)]", + "diffDiskSettings": { + "option": "local", + "placement": "[parameters('node').disk.ephemeral_disk_placement_type]" + }, + "caching": "ReadOnly", + "createOption": "FromImage", + "diskSizeGB": "[parameters('node').osdisk_size_in_gb]" + } + } + }, + "getCreateDisk": { + "parameters": [ + { + "type": "object", + "name": "disk" + }, + { + "type": "string", + "name": "diskName" + }, + { + "type": "int", + "name": "index" + } + ], + "output": { + "type": "object", + "value": { + "name": "[parameters('diskName')]", + "createOption": "[parameters('disk').create_option]", + "caching": "[parameters('disk').caching_type]", + "diskSizeGB": "[parameters('disk').size]", + "lun": "[parameters('index')]", + "managedDisk": { + "storageAccountType": "[parameters('disk').type]" + } + } + } + }, + "getAttachDisk": { + "parameters": [ + { + "type": "object", + "name": "disk" + }, + { + "type": "string", + "name": "diskName" + }, + { + "type": "int", + "name": "index" + } + ], + "output": { + "type": "object", + "value": { + "lun": "[parameters('index')]", + "createOption": "attach", + "caching": "[parameters('disk').caching_type]", + "managedDisk": { + "id": "[resourceId('Microsoft.Compute/disks', parameters('diskName'))]" + } + } + } + }, + "getDataDisk": { + "parameters": [ + { + "type": "string", + "name": "nodeName" + }, + { + "type": "object", + "name": "dataDisk" + }, + { + "type": "int", + "name": "index" + } + ], + "output": { + "type": "object", + "value": "[if(equals(parameters('dataDisk').type, 'UltraSSD_LRS'), __bicep.getAttachDisk(parameters('dataDisk'), format('{0}-data-disk-{1}', parameters('nodeName'), parameters('index')), parameters('index')), __bicep.getCreateDisk(parameters('dataDisk'), format('{0}-data-disk-{1}', parameters('nodeName'), parameters('index')), parameters('index')))]" + } + }, + "getOsDiskSharedGallery": { + "parameters": [ + { + "type": "object", + "name": "shared_gallery" + } + ], + "output": { + "type": "object", + "value": { + "id": "[resourceId(parameters('shared_gallery').subscription_id, if(empty(parameters('shared_gallery').resource_group_name), 'None', parameters('shared_gallery').resource_group_name), 'Microsoft.Compute/galleries/images/versions', parameters('shared_gallery').image_gallery, parameters('shared_gallery').image_definition, parameters('shared_gallery').image_version)]" + } + } + }, + "getOSDiskCommunityGalleryImage": { + "parameters": [ + { + "type": "object", + "name": "community_gallery_image" + } + ], + "output": { + "type": "object", + "value": { + "communityGalleryImageId": "[format('/CommunityGalleries/{0}/Images/{1}/Versions/{2}', parameters('community_gallery_image').image_gallery, parameters('community_gallery_image').image_definition, parameters('community_gallery_image').image_version)]" + } + } + }, + "getOsDiskMarketplace": { + "parameters": [ + { + "type": "object", + "name": "marketplace" + } + ], + "output": { + "type": "object", + "value": { + "publisher": "[parameters('marketplace').publisher]", + "offer": "[parameters('marketplace').offer]", + "sku": "[parameters('marketplace').sku]", + "version": "[parameters('marketplace').version]" + } + } + }, + "generateImageReference": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "object", + "value": "[if(__bicep.isVhd(parameters('node')), __bicep.getOsDiskVhd(parameters('node').name), if(not(empty(parameters('node').shared_gallery)), __bicep.getOsDiskSharedGallery(parameters('node').shared_gallery), if(not(empty(parameters('node').community_gallery_image)), __bicep.getOSDiskCommunityGalleryImage(parameters('node').community_gallery_image), __bicep.getOsDiskMarketplace(parameters('node').marketplace))))]" + } + }, + "getSecurityProfileForOSDisk": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "object", + "value": "[if(empty(parameters('node').security_profile.disk_encryption_set_id), createObject('securityEncryptionType', parameters('node').security_profile.encryption_type), createObject('securityEncryptionType', parameters('node').security_profile.encryption_type, 'diskEncryptionSet', createObject('id', parameters('node').security_profile.disk_encryption_set_id)))]" + } + }, + "generateOsProfile": { + "parameters": [ + { + "type": "object", + "name": "node" + }, + { + "type": "string", + "name": "admin_username" + }, + { + "type": "string", + "name": "admin_password" + }, + { + "type": "string", + "name": "admin_key_data" + } + ], + "output": { + "type": "object", + "value": { + "computername": "[parameters('node').short_name]", + "adminUsername": "[parameters('admin_username')]", + "adminPassword": "[if(empty(parameters('admin_password')), null(), parameters('admin_password'))]", + "linuxConfiguration": "[if(and(not(empty(parameters('admin_key_data'))), parameters('node').is_linux), __bicep.getLinuxConfiguration(format('/home/{0}/.ssh/authorized_keys', parameters('admin_username')), parameters('admin_key_data'), empty(parameters('admin_password'))), null())]" + } + } + }, + "generateSecurityProfile": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "object", + "value": { + "uefiSettings": { + "secureBootEnabled": "[parameters('node').security_profile.secure_boot]", + "vTpmEnabled": true + }, + "securityType": "[parameters('node').security_profile.security_type]" + } + } + }, + "getOsProfile": { + "parameters": [ + { + "type": "object", + "name": "node" + }, + { + "type": "string", + "name": "admin_username" + }, + { + "type": "string", + "name": "admin_password" + }, + { + "type": "string", + "name": "admin_key_data" + } + ], + "output": { + "type": "object", + "nullable": true, + "value": "[if(__bicep.isCvm(parameters('node')), null(), __bicep.generateOsProfile(parameters('node'), parameters('admin_username'), parameters('admin_password'), parameters('admin_key_data')))]" + } + }, + "getImageReference": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "object", + "nullable": true, + "value": "[if(__bicep.isCvm(parameters('node')), null(), __bicep.generateImageReference(parameters('node')))]" + } + }, + "getSecurityProfile": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "object", + "nullable": true, + "value": "[if(empty(parameters('node').security_profile), null(), __bicep.generateSecurityProfile(parameters('node')))]" + } + }, + "getOSImage": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "object", + "value": { + "name": "[format('{0}-osDisk', parameters('node').name)]", + "managedDisk": { + "storageAccountType": "[parameters('node').os_disk_type]", + "securityProfile": "[if(or(empty(parameters('node').security_profile), not(equals(parameters('node').security_profile.security_type, 'ConfidentialVM'))), null(), __bicep.getSecurityProfileForOSDisk(parameters('node')))]" + }, + "caching": "[if(equals(parameters('node').os_disk_type, 'Ephemeral'), 'ReadOnly', 'ReadWrite')]", + "createOption": "FromImage", + "diskSizeGB": "[parameters('node').osdisk_size_in_gb]" + } + } + }, + "getVMOsDisk": { + "parameters": [ + { + "type": "object", + "name": "node" + } + ], + "output": { + "type": "object", + "value": "[if(__bicep.isCvm(parameters('node')), __bicep.getOSDisk(format('{0}-disk', parameters('node').name)), if(equals(parameters('node').os_disk_type, 'Ephemeral'), __bicep.getEphemeralOSImage(parameters('node')), __bicep.getOSImage(parameters('node'))))]" + } + }, + "getAvailabilitySetId": { + "parameters": [ + { + "type": "string", + "name": "availability_set_name" + } + ], + "output": { + "type": "object", + "value": { + "id": "[resourceId('Microsoft.Compute/availabilitySets', parameters('availability_set_name'))]" + } + } + } + } + } + ], + "parameters": { + "vhd_storage_name": { + "type": "string", + "metadata": { + "description": "storage name for copied vhds" + } + }, + "location": { + "type": "string", + "metadata": { + "description": "location" + } + }, + "nodes": { + "type": "array", + "metadata": { + "description": "all nodes" + } + }, + "admin_username": { + "type": "string", + "metadata": { + "description": "user name" + } + }, + "admin_password": { + "type": "string", + "metadata": { + "description": "password" + } + }, + "admin_key_data": { + "type": "string", + "metadata": { + "description": "public key data" + } + }, + "shared_resource_group_name": { + "type": "string", + "metadata": { + "description": "the name of shared resource group" + } + }, + "subnet_count": { + "type": "int", + "metadata": { + "description": "created subnet count" + } + }, + "availability_options": { + "type": "object", + "metadata": { + "description": "options for availability sets, zones, and VMSS" + } + }, + "virtual_network_resource_group": { + "type": "string", + "metadata": { + "description": "the name of vnet resource group" + } + }, + "virtual_network_name": { + "type": "string", + "metadata": { + "description": "the name of vnet" + } + }, + "subnet_prefix": { + "type": "string", + "metadata": { + "description": "the prefix of the subnets" + } + }, + "vm_tags": { + "type": "object", + "metadata": { + "description": "tags of virtual machine" + } + }, + "tags": { + "type": "object", + "metadata": { + "description": "tags of azure resources" + } + }, + "data_disks": { + "type": "array", + "metadata": { + "description": "data disk array." + } + }, + "is_ultradisk": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "whether to use ultra disk" + } + }, + "ip_service_tags": { + "type": "object", + "metadata": { + "description": "IP Service Tags" + } + } + }, + "variables": { + "copy": [ + { + "name": "ip_tags", + "count": "[length(objectKeys(parameters('ip_service_tags')))]", + "input": { + "ipTagType": "[objectKeys(parameters('ip_service_tags'))[copyIndex('ip_tags')]]", + "tag": "[parameters('ip_service_tags')[objectKeys(parameters('ip_service_tags'))[copyIndex('ip_tags')]]]" + } + } + ], + "vnet_id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('virtual_network_name'))]", + "node_count": "[length(parameters('nodes'))]", + "availability_set_name_value": "lisa-availabilitySet", + "existing_subnet_ref": "[if(empty(parameters('virtual_network_resource_group')), '', resourceId(parameters('virtual_network_resource_group'), 'Microsoft.Network/virtualNetworks/subnets', parameters('virtual_network_name'), parameters('subnet_prefix')))]", + "availability_set_tags": "[parameters('availability_options').availability_set_tags]", + "availability_set_properties": "[parameters('availability_options').availability_set_properties]", + "availability_zones": "[parameters('availability_options').availability_zones]", + "availability_type": "[parameters('availability_options').availability_type]", + "use_availability_set": "[equals(variables('availability_type'), 'availability_set')]", + "use_availability_zones": "[equals(variables('availability_type'), 'availability_zone')]", + "availability_set_value": "[if(variables('use_availability_set'), __bicep.getAvailabilitySetId(variables('availability_set_name_value')), null())]", + "combined_vm_tags": "[union(parameters('tags'), parameters('vm_tags'))]", + "combined_aset_tags": "[union(parameters('tags'), variables('availability_set_tags'))]" + }, + "resources": { + "virtual_network_name_resource": { + "condition": "[empty(parameters('virtual_network_resource_group'))]", + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2020-05-01", + "name": "[parameters('virtual_network_name')]", + "tags": "[parameters('tags')]", + "location": "[parameters('location')]", + "properties": { + "copy": [ + { + "name": "subnets", + "count": "[length(range(0, parameters('subnet_count')))]", + "input": { + "name": "[format('{0}{1}', parameters('subnet_prefix'), range(0, parameters('subnet_count'))[copyIndex('subnets')])]", + "properties": { + "addressPrefix": "[format('10.0.{0}.0/24', range(0, parameters('subnet_count'))[copyIndex('subnets')])]" + } + } + } + ], + "addressSpace": { + "addressPrefixes": [ + "10.0.0.0/16" + ] + } + } + }, + "availability_set": { + "condition": "[variables('use_availability_set')]", + "type": "Microsoft.Compute/availabilitySets", + "apiVersion": "2019-07-01", + "name": "[variables('availability_set_name_value')]", + "location": "[parameters('location')]", + "tags": "[variables('combined_aset_tags')]", + "sku": { + "name": "Aligned" + }, + "properties": "[variables('availability_set_properties')]" + }, + "nodes_public_ip": { + "copy": { + "name": "nodes_public_ip", + "count": "[length(range(0, variables('node_count')))]" + }, + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2020-05-01", + "name": "[format('{0}-public-ip', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name)]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "properties": { + "publicIPAllocationMethod": "[if(or(parameters('is_ultradisk'), variables('use_availability_zones')), 'Static', 'Dynamic')]", + "ipTags": "[if(empty(variables('ip_tags')), null(), variables('ip_tags'))]" + }, + "sku": { + "name": "[if(or(parameters('is_ultradisk'), variables('use_availability_zones')), 'Standard', 'Basic')]" + }, + "zones": "[if(variables('use_availability_zones'), variables('availability_zones'), null())]" + }, + "nodes_image": { + "copy": { + "name": "nodes_image", + "count": "[length(range(0, variables('node_count')))]" + }, + "condition": "[and(__bicep.isVhd(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]]), empty(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].vhd.vmgs_path))]", + "type": "Microsoft.Compute/images", + "apiVersion": "2019-03-01", + "name": "[format('{0}-image', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name)]", + "tags": "[parameters('tags')]", + "location": "[parameters('location')]", + "properties": { + "storageProfile": { + "osDisk": { + "osType": "Linux", + "osState": "Generalized", + "blobUri": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].vhd.vhd_path]", + "storageAccountType": "Standard_LRS" + } + }, + "hyperVGeneration": "[format('V{0}', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].hyperv_generation)]" + } + }, + "nodes_disk": { + "copy": { + "name": "nodes_disk", + "count": "[length(range(0, variables('node_count')))]" + }, + "condition": "[__bicep.isCvm(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]])]", + "type": "Microsoft.Compute/disks", + "apiVersion": "2021-04-01", + "name": "[format('{0}-disk', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name)]", + "tags": "[parameters('tags')]", + "location": "[parameters('location')]", + "sku": { + "name": "Standard_LRS" + }, + "properties": { + "osType": "Linux", + "hyperVGeneration": "[format('V{0}', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].hyperv_generation)]", + "securityProfile": { + "securityType": "ConfidentialVM_VMGuestStateOnlyEncryptedWithPlatformKey" + }, + "creationData": { + "createOption": "ImportSecure", + "storageAccountId": "[resourceId(parameters('shared_resource_group_name'), 'Microsoft.Storage/storageAccounts', parameters('vhd_storage_name'))]", + "securityDataUri": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].vhd.vmgs_path]", + "sourceUri": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].vhd.vhd_path]" + } + }, + "zones": "[if(variables('use_availability_zones'), variables('availability_zones'), null())]" + }, + "nodes_data_disks": { + "copy": { + "name": "nodes_data_disks", + "count": "[length(range(0, mul(length(parameters('data_disks')), variables('node_count'))))]" + }, + "condition": "[parameters('is_ultradisk')]", + "type": "Microsoft.Compute/disks", + "apiVersion": "2022-03-02", + "name": "[format('{0}-data-disk-{1}', parameters('nodes')[div(range(0, mul(length(parameters('data_disks')), variables('node_count')))[copyIndex()], length(parameters('data_disks')))].name, mod(range(0, mul(length(parameters('data_disks')), variables('node_count')))[copyIndex()], length(parameters('data_disks'))))]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "properties": { + "diskSizeGB": "[parameters('data_disks')[mod(range(0, mul(length(parameters('data_disks')), variables('node_count')))[copyIndex()], length(parameters('data_disks')))].size]", + "creationData": { + "createOption": "[parameters('data_disks')[mod(range(0, mul(length(parameters('data_disks')), variables('node_count')))[copyIndex()], length(parameters('data_disks')))].create_option]" + }, + "diskIOPSReadWrite": "[parameters('data_disks')[mod(range(0, mul(length(parameters('data_disks')), variables('node_count')))[copyIndex()], length(parameters('data_disks')))].iops]", + "diskMBpsReadWrite": "[parameters('data_disks')[mod(range(0, mul(length(parameters('data_disks')), variables('node_count')))[copyIndex()], length(parameters('data_disks')))].throughput]" + }, + "sku": { + "name": "[parameters('data_disks')[mod(range(0, mul(length(parameters('data_disks')), variables('node_count')))[copyIndex()], length(parameters('data_disks')))].type]" + }, + "zones": "[if(variables('use_availability_zones'), variables('availability_zones'), null())]" + }, + "nodes_vms": { + "copy": { + "name": "nodes_vms", + "count": "[length(range(0, variables('node_count')))]" + }, + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2024-03-01", + "name": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name]", + "location": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].location]", + "tags": "[variables('combined_vm_tags')]", + "plan": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].purchase_plan]", + "properties": { + "availabilitySet": "[variables('availability_set_value')]", + "hardwareProfile": { + "vmSize": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].vm_size]" + }, + "osProfile": "[__bicep.getOsProfile(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]], parameters('admin_username'), parameters('admin_password'), parameters('admin_key_data'))]", + "storageProfile": { + "copy": [ + { + "name": "dataDisks", + "count": "[length(parameters('data_disks'))]", + "input": "[__bicep.getDataDisk(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name, parameters('data_disks')[copyIndex('dataDisks')], copyIndex('dataDisks'))]" + } + ], + "imageReference": "[__bicep.getImageReference(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]])]", + "osDisk": "[__bicep.getVMOsDisk(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]])]", + "diskControllerType": "[if(equals(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].disk_controller_type, 'SCSI'), null(), parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].disk_controller_type)]" + }, + "networkProfile": { + "copy": [ + { + "name": "networkInterfaces", + "count": "[length(range(0, parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].nic_count))]", + "input": { + "id": "[resourceId('Microsoft.Network/networkInterfaces', format('{0}-nic-{1}', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name, range(0, parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].nic_count)[copyIndex('networkInterfaces')]))]", + "properties": { + "primary": "[if(equals(range(0, parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].nic_count)[copyIndex('networkInterfaces')], 0), true(), false())]" + } + } + } + ] + }, + "diagnosticsProfile": { + "bootDiagnostics": { + "enabled": true + } + }, + "additionalCapabilities": { + "ultraSSDEnabled": "[if(equals(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].data_disk_type, 'UltraSSD_LRS'), true(), false())]" + }, + "securityProfile": "[__bicep.getSecurityProfile(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]])]" + }, + "zones": "[if(variables('use_availability_zones'), variables('availability_zones'), null())]", + "dependsOn": [ + "availability_set", + "nodes_disk", + "nodes_image", + "nodes_nics", + "virtual_network_name_resource" + ] + }, + "nodes_nics": { + "copy": { + "name": "nodes_nics", + "count": "[length(range(0, variables('node_count')))]" + }, + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-nics', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name)]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "vmName": { + "value": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name]" + }, + "nic_count": { + "value": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].nic_count]" + }, + "location": { + "value": "[parameters('location')]" + }, + "vnet_id": { + "value": "[variables('vnet_id')]" + }, + "subnet_prefix": { + "value": "[parameters('subnet_prefix')]" + }, + "existing_subnet_ref": { + "value": "[variables('existing_subnet_ref')]" + }, + "enable_sriov": { + "value": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].enable_sriov]" + }, + "tags": { + "value": "[parameters('tags')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "languageVersion": "2.0", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "7856159159103188049" + } + }, + "functions": [ + { + "namespace": "__bicep", + "members": { + "getPublicIpAddress": { + "parameters": [ + { + "type": "string", + "name": "vmName" + } + ], + "output": { + "type": "object", + "value": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', format('{0}-public-ip', parameters('vmName')))]" + } + } + } + } + } + ], + "parameters": { + "vmName": { + "type": "string" + }, + "nic_count": { + "type": "int" + }, + "location": { + "type": "string" + }, + "vnet_id": { + "type": "string" + }, + "subnet_prefix": { + "type": "string" + }, + "existing_subnet_ref": { + "type": "string" + }, + "enable_sriov": { + "type": "bool" + }, + "tags": { + "type": "object" + } + }, + "variables": { + "publicIpAddress": "[__bicep.getPublicIpAddress(parameters('vmName'))]" + }, + "resources": { + "vm_nics": { + "copy": { + "name": "vm_nics", + "count": "[length(range(0, parameters('nic_count')))]" + }, + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2023-06-01", + "name": "[format('{0}-nic-{1}', parameters('vmName'), range(0, parameters('nic_count'))[copyIndex()])]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "properties": { + "ipConfigurations": [ + { + "name": "IPv4Config", + "properties": { + "privateIPAddressVersion": "IPv4", + "publicIPAddress": "[if(equals(0, range(0, parameters('nic_count'))[copyIndex()]), variables('publicIpAddress'), null())]", + "subnet": { + "id": "[if(not(empty(parameters('existing_subnet_ref'))), parameters('existing_subnet_ref'), format('{0}/subnets/{1}{2}', parameters('vnet_id'), parameters('subnet_prefix'), range(0, parameters('nic_count'))[copyIndex()]))]" + }, + "privateIPAllocationMethod": "Dynamic" + } + } + ], + "enableAcceleratedNetworking": "[parameters('enable_sriov')]" + } + } + } + } + }, + "dependsOn": [ + "[format('nodes_public_ip[{0}]', range(0, variables('node_count'))[copyIndex()])]", + "virtual_network_name_resource" + ] + } + } +} \ No newline at end of file diff --git a/lisa/sut_orchestrator/azure/autogen_arm_template.json b/lisa/sut_orchestrator/azure/autogen_arm_template.json index 18c4a3484c..5f8a67eba4 100644 --- a/lisa/sut_orchestrator/azure/autogen_arm_template.json +++ b/lisa/sut_orchestrator/azure/autogen_arm_template.json @@ -5,8 +5,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.30.23.60470", - "templateHash": "17909783643222378721" + "version": "0.32.4.45862", + "templateHash": "16398577375970436728" } }, "functions": [ @@ -113,7 +113,7 @@ "name": "[format('{0}-osDisk', parameters('node').name)]", "diffDiskSettings": { "option": "local", - "placement": "CacheDisk" + "placement": "[parameters('node').ephemeral.ephemeral_disk_placement_type]" }, "caching": "ReadOnly", "createOption": "FromImage", @@ -685,7 +685,7 @@ "count": "[length(range(0, variables('node_count')))]" }, "type": "Microsoft.Compute/virtualMachines", - "apiVersion": "2022-08-01", + "apiVersion": "2024-03-01", "name": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name]", "location": "[parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].location]", "tags": "[variables('combined_vm_tags')]", @@ -787,8 +787,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.30.23.60470", - "templateHash": "12249187708601787514" + "version": "0.32.4.45862", + "templateHash": "7856159159103188049" } }, "functions": [ diff --git a/lisa/sut_orchestrator/azure/common.py b/lisa/sut_orchestrator/azure/common.py index 58d6921cb8..9162434860 100644 --- a/lisa/sut_orchestrator/azure/common.py +++ b/lisa/sut_orchestrator/azure/common.py @@ -8,6 +8,7 @@ import sys from dataclasses import InitVar, dataclass, field from datetime import datetime, timedelta, timezone +from enum import Enum from functools import lru_cache, partial from pathlib import Path, PurePath from threading import Lock @@ -1068,6 +1069,7 @@ class AzureNodeArmParameter(AzureNodeSchema): os_disk_type: str = "" data_disk_type: str = "" disk_controller_type: str = "" + ephemeral_disk: str = "" security_profile: Dict[str, Any] = field(default_factory=dict) @classmethod @@ -1104,6 +1106,25 @@ def get_create_option() -> List[str]: ] +# EphemeralOSDiskPlacements +# refer +# https://learn.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks-faq +class DiskPlacementType(str, Enum): + NONE = "" + RESOURCE = "ResourceDisk" + CACHE = "CacheDisk" + NVME = "NvmeDisk" + + +def get_disk_placement_priority() -> List[DiskPlacementType]: + return [ + DiskPlacementType.NONE, + DiskPlacementType.RESOURCE, + DiskPlacementType.CACHE, + DiskPlacementType.NVME, + ] + + @dataclass_json() @dataclass class DataDiskSchema: diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index ed0e86bdcd..008c775b09 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -97,6 +97,7 @@ AzureCapability, AzureImageSchema, AzureNodeSchema, + DiskPlacementType, check_or_create_storage_account, create_update_private_dns_zone_groups, create_update_private_endpoints, @@ -112,6 +113,7 @@ delete_virtual_network_links, find_by_name, get_compute_client, + get_disk_placement_priority, get_network_client, get_node_context, get_or_create_file_share, @@ -1307,6 +1309,31 @@ def get_all_nics_ip_info(self) -> List[IpInfo]: @dataclass() class AzureDiskOptionSettings(schema.DiskOptionSettings): has_resource_disk: Optional[bool] = None + ephemeral_disk_placement_type: Optional[ + Union[search_space.SetSpace[DiskPlacementType], DiskPlacementType] + ] = field( # type:ignore + default_factory=partial( + search_space.SetSpace, + items=[ + DiskPlacementType.NONE, + DiskPlacementType.RESOURCE, + DiskPlacementType.CACHE, + DiskPlacementType.NVME, + ], + ), + metadata=field_metadata( + decoder=partial( + search_space.decode_nullable_set_space, + base_type=DiskPlacementType, + default_values=[ + DiskPlacementType.NONE, + DiskPlacementType.RESOURCE, + DiskPlacementType.CACHE, + DiskPlacementType.NVME, + ], + ) + ), + ) def __hash__(self) -> int: return super().__hash__() @@ -1373,6 +1400,13 @@ def check(self, capability: Any) -> search_space.ResultReason: ), "has_resource_disk", ) + result.merge( + search_space.check_setspace( + self.ephemeral_disk_placement_type, + capability.ephemeral_disk_placement_type, + ), + "ephemeral_disk_placement_type", + ) result.merge( search_space.check_countspace( self.max_data_disk_count, capability.max_data_disk_count @@ -1475,6 +1509,34 @@ def _call_requirement_method( schema.disk_controller_type_priority, ) + # refer + # https://learn.microsoft.com/en-us/powershell/module/az.compute/set-azvmssstorageprofile?view=azps-13.0.0 # noqa: E501 + # https://github.com/MicrosoftDocs/azure-compute-docs/blob/main/articles/virtual-machines/ephemeral-os-disks-faq.md # noqa: E501 + if value.os_disk_type == schema.DiskType.Ephemeral: + cap_ephemeral_disk_placement_type = capability.ephemeral_disk_placement_type + if isinstance(cap_ephemeral_disk_placement_type, search_space.SetSpace): + assert len(cap_ephemeral_disk_placement_type) > 0, ( + "capability should have at least one ephemeral disk placement type," + " but it's empty" + ) + elif isinstance(cap_ephemeral_disk_placement_type, DiskPlacementType): + cap_ephemeral_disk_placement_type = search_space.SetSpace[ + DiskPlacementType + ](is_allow_set=True, items=[cap_ephemeral_disk_placement_type]) + else: + raise LisaException( + "unknown ephemeral disk placement type " + f"on capability, type: {cap_ephemeral_disk_placement_type}" + ) + + value.ephemeral_disk_placement_type = getattr( + search_space, f"{method.value}_setspace_by_priority" + )( + self.ephemeral_disk_placement_type, + capability.ephemeral_disk_placement_type, + get_disk_placement_priority(), + ) + # below values affect data disk only. if self.data_disk_count is not None or capability.data_disk_count is not None: value.data_disk_count = getattr(search_space, f"{method.value}_countspace")( @@ -2019,6 +2081,11 @@ def _get_raw_data_disks_bsd(self) -> List[str]: return data_disks + def get_ephemeral_disk_placement_type(self) -> Any: + azure_platform: AzurePlatform = self._platform # type: ignore + vm = get_vm(azure_platform, self._node) + return vm.storage_profile.os_disk.diff_disk_settings.placement + def get_azure_disk_type(disk_type: schema.DiskType) -> str: assert isinstance(disk_type, schema.DiskType), ( diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index 5e58639058..cbb562587e 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -104,12 +104,14 @@ CommunityGalleryImageSchema, DataDiskCreateOption, DataDiskSchema, + DiskPlacementType, SharedImageGallerySchema, check_or_create_resource_group, check_or_create_storage_account, convert_to_azure_node_space, get_compute_client, get_deployable_vhd_path, + get_disk_placement_priority, get_environment_context, get_marketplace_ordering_client, get_node_context, @@ -1417,6 +1419,7 @@ def _create_node_arm_parameters( arm_parameters.os_disk_type = features.get_azure_disk_type( capability.disk.os_disk_type ) + assert isinstance(capability.disk.data_disk_type, schema.DiskType) arm_parameters.data_disk_type = features.get_azure_disk_type( capability.disk.data_disk_type @@ -1433,6 +1436,18 @@ def _create_node_arm_parameters( ) arm_parameters.disk_controller_type = capability.disk.disk_controller_type.value + assert isinstance( + features.AzureDiskOptionSettings( + capability.disk + ).ephemeral_disk_placement_type, + DiskPlacementType, + ) + arm_parameters.ephemeral_disk = ( + features.AzureDiskOptionSettings( + capability.disk + ).ephemeral_disk_placement_type.value + ) + assert capability.network_interface assert isinstance( capability.network_interface.nic_count, int @@ -1794,14 +1809,41 @@ def _resource_sku_to_capability( # noqa: C901 else: node_space.disk.disk_controller_type.add(schema.DiskControllerType.SCSI) + # If EphemeralOSDisk is supported, then check for the placement type if azure_raw_capabilities.get("EphemeralOSDiskSupported", None) == "True": - # Check if CachedDiskBytes is greater than 30GB - # We use diff disk as cache disk for ephemeral OS disk + node_space.disk.os_disk_type.add(schema.DiskType.Ephemeral) + # Add the EphemeralDiskPlacementType + ephemeral_disk_placement_types = azure_raw_capabilities.get( + "SupportedEphemeralOSDiskPlacements", None + ) + if ephemeral_disk_placement_types: + for allowed_type in ephemeral_disk_placement_types.split(","): + try: + node_space.disk.ephemeral_disk_placement_type.add( + DiskPlacementType(allowed_type) + ) + except ValueError: + self._log.error( + f"'{allowed_type}' is not a known Ephemeral Disk Placement" + f" Type " + f"({[x for x in DiskPlacementType]})" + ) + + # EphemeralDiskPlacementType can be - ResourceDisk, CacheDisk or NvmeDisk. + # Depending on that, "CachedDiskBytes" may or may not be found in + # capabilities. + # refer + # https://learn.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks-faq + resource_disk_bytes = azure_raw_capabilities.get("MaxResourceVolumeMB", 0) cached_disk_bytes = azure_raw_capabilities.get("CachedDiskBytes", 0) - cached_disk_bytes_gb = int(int(cached_disk_bytes) / 1024 / 1024 / 1024) - if cached_disk_bytes_gb >= 30: - node_space.disk.os_disk_type.add(schema.DiskType.Ephemeral) - node_space.disk.os_disk_size = cached_disk_bytes_gb + nvme_disk_bytes = azure_raw_capabilities.get("NvmeDiskSizeInMiB", 0) + if nvme_disk_bytes: + node_space.disk.os_disk_size = int(int(nvme_disk_bytes) / 1024) + elif cached_disk_bytes: + node_space.disk.os_disk_size = int( + int(cached_disk_bytes) / 1024 / 1024 / 1024) + else: + node_space.disk.os_disk_size = int(int(resource_disk_bytes) / 1024) # set AN if azure_raw_capabilities.get("AcceleratedNetworkingEnabled", None) == "True": @@ -2754,6 +2796,14 @@ def _set_disk_features( isinstance(node_space.disk.os_disk_type, search_space.SetSpace) and node_space.disk.os_disk_type.isunique(schema.DiskType.Ephemeral) ): + node_space.disk = features.AzureDiskOptionSettings(node_space.disk) + node_space.disk.ephemeral_disk_placement_type = search_space.SetSpace[ + DiskPlacementType + ]( + is_allow_set=True, + items=[node_space.disk.ephemeral_disk_placement_type], + ) + node_space.disk.os_disk_size = search_space.IntRange( min=self._get_os_disk_size(azure_runbook) )