adding fixes for gen3-helper-test-suite (#2700)

* adding fixes for gen3-helper-test-suite * testing changing the test case to look at server2 for snapshots * updating "expected results" to match the changes we made in cloud-auto. * passing in "bucketName" for the modified s3_info function. Also, created a new bucket as "real-bucket-ci" since "real-bucket" does not exist and the terraform was broken for that bucket. * removing authz test as it is not part of the GH integration tests. * removing this portion as was part of the old testing framework.
uc-cdis · Jan 16, 2025 · 7563589 · 7563589
1 parent cd71d6f
commit 7563589
Show file tree

Hide file tree

Showing 8 changed files with 16 additions and 57 deletions.
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -158,22 +158,6 @@
         "line_number": 5
       }
     ],
-    "Jenkinsfile": [
-      {
-        "type": "Secret Keyword",
-        "filename": "Jenkinsfile",
-        "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf",
-        "is_verified": false,
-        "line_number": 144
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "Jenkinsfile",
-        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
-        "is_verified": false,
-        "line_number": 147
-      }
-    ],
     "ansible/roles/slurm/README.md": [
       {
         "type": "Base64 High Entropy String",
@@ -3253,5 +3237,5 @@
       }
     ]
   },
-  "generated_at": "2024-11-25T11:39:59Z"
+  "generated_at": "2025-01-16T15:40:00Z"
 }
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -433,39 +433,13 @@ spec:
             }
         }
 
-        stage('authzTest') {
-            steps {
-                script {
-                    try {
-                        if(!doNotRunTests) {
-                            // test revproxy+arborist /gen3-authz stuff
-                            kubeHelper.kube(kubectlNamespace, {
-                                sh('bash cloud-automation/gen3/bin/testsuite.sh --filter authz');
-                            });
-                        } else {
-                            Utils.markStageSkippedForConditional(STAGE_NAME)
-                        }
-                    } catch (ex) {
-                        metricsHelper.writeMetricWithResult(STAGE_NAME, false)
-                        pipelineHelper.handleError(ex)
-                    }
-                    metricsHelper.writeMetricWithResult(STAGE_NAME, true)
-                }
-            }
-        }
     }
     post {
         always {
             script {
                 kubeHelper.teardown(kubeLocks)
                 testHelper.teardown(doNotRunTests)
                 pipelineHelper.teardown(currentBuild.result)
-                if(!skipUnitTests) {
-                    // tear down network policies deployed by the tests
-                    kubeHelper.kube(kubectlNamespace, {
-                        sh(script: 'kubectl --namespace="' + kubectlNamespace + '" delete networkpolicies --all', returnStatus: true);
-                    });
-                }
             }
         }
     }

diff --git a/gen3/bin/db.sh b/gen3/bin/db.sh
@@ -394,7 +394,7 @@ gen3_db_namespace() {
 #
 # Given a gen3 server name, determine the RDS instance id
 #
-gen3_db_server_rds_id() {
+gen3_db_server_aurora_cluster() {
   local address
   local serverInfo
 
@@ -406,7 +406,7 @@ gen3_db_server_rds_id() {
     gen3_log_err "unable to determine address for $@"
     return 1
   fi
-  aws rds describe-db-instances | jq -e -r --arg address "$address" '.DBInstances[] | select(.Endpoint.Address==$address) | .DBInstanceIdentifier'
+  aws rds describe-db-instances | jq -e -r --arg address "$address" '.DBInstances[] | select(.Endpoint.Address==$address) | .DBClusterIdentifier'
 }
 
 #
@@ -431,15 +431,15 @@ gen3_db_snapshot_take() {
     dryRun=true
   fi
   local instanceId
-  if ! instanceId="$(gen3_db_server_rds_id "$serverName")"; then
+  if ! instanceId="$(gen3_db_server_aurora_cluster "$serverName")"; then
     gen3_log_err "failed to find rds instance id for server: $serverName"
     return 1
   fi
   snapshotId="gen3-snapshot-${serverName}-$(date -u +%Y%m%d-%H%M%S)"
   if [[ "$dryRun" == true ]]; then
     gen3_log_info "dryrun mode - not taking snapshot"
   else
-    aws rds create-db-snapshot --db-snapshot-identifier "$snapshotId" --db-instance-identifier "$instanceId"
+    aws rds create-db-cluster-snapshot --db-cluster-snapshot-identifier "$snapshotId" --db-cluster-identifier "$instanceId"
   fi
 }
 
@@ -457,11 +457,11 @@ gen3_db_snapshot_list() {
     return 1
   fi
   local instanceId
-  if ! instanceId="$(gen3_db_server_rds_id "$serverName")"; then
+  if ! instanceId="$(gen3_db_server_aurora_cluster "$serverName")"; then
     gen3_log_err "failed to find rds instance id for server: $serverName"
     return 1
   fi
-  aws rds describe-db-snapshots --db-instance-identifier "$instanceId"
+  aws rds describe-db-cluster-snapshots --db-cluster-identifier "$instanceId"
 }
 
 

diff --git a/gen3/bin/s3.sh b/gen3/bin/s3.sh
@@ -169,6 +169,7 @@ gen3_s3_info() {
   local writerName="bucket_writer_$1"
   local readerName="bucket_reader_$1"
   local AWS_ACCOUNT_ID=$(gen3_aws_run aws sts get-caller-identity | jq -r .Account)
+  local bucketName=$1
 
   if [[ -z "$AWS_ACCOUNT_ID" ]]; then
     gen3_log_err "Unable to fetch AWS account ID."

diff --git a/gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml b/gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml
@@ -29,8 +29,8 @@ spec:
         netnolimit: "yes"
         public: "yes"
         userhelper: "yes"
-        service: "fence"
         env: null
+        hostname: null
         version: 'master'
         date: "1579711361"
     spec:
@@ -273,4 +273,4 @@ spec:
               fi
             else
               echo "Db migration not available in this version of fence"
-            fi
+            fi
diff --git a/gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml b/gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml
@@ -23,10 +23,10 @@ spec:
         public: "yes"
         # to download dictionary
         s3: "yes"
-        service: "sheepdog"
         env: null
         version: 'master'
         date: "1522344212"
+        hostname: null
     spec:
       affinity:
         podAntiAffinity:
@@ -173,4 +173,4 @@ spec:
               cpu: 100m
               memory: 200Mi
             limits:
-              memory: 800Mi
+              memory: 800Mi
diff --git a/gen3/test/dbTest.sh b/gen3/test/dbTest.sh
@@ -87,15 +87,15 @@ test_db_services() {
 
 test_db_snapshot_list() {
   local snapshotJson
-  snapshotJson="$(gen3 db snapshot list server1)"; because $? "gen3 db snapshot list server1 should work"
+  snapshotJson="$(gen3 db snapshot list server2)"; because $? "gen3 db snapshot list server2 should work"
   local snapCount
-  snapCount="$(jq -e -r '.DBSnapshots | length' <<<"$snapshotJson")"; 
+  snapCount="$(jq -e -r '.DBClusterSnapshots | length' <<<"$snapshotJson")"; 
     because $? "snap list json has expected structure"
   [[ "$snapCount" =~ ^[0-9]+$ && "$snapCount" -gt 0 ]]; because $? "server1 has at least 1 snapshot"
 }
 
 test_db_snapshot_take() {
-  gen3 db snapshot take server1 --dryrun; because $? "gen3 db snapshot take server1 should work"
+  gen3 db snapshot take server2 --dryrun; because $? "gen3 db snapshot take server2 should work"
 }
 
 test_db_backup_restore() {

diff --git a/gen3/test/s3Test.sh b/gen3/test/s3Test.sh
@@ -59,7 +59,7 @@ test_s3_info() {
   }
 
   ! gen3_s3_info bogus-bucket; because $? "when bucket doesn't exist it should fail"
-  policies=$(gen3_s3_info real-bucket); because $? "when bucket and policies exist it should succeed" 
+  policies=$(gen3_s3_info real-bucket-ci); because $? "when bucket and policies exist it should succeed" 
   readOnly="$(echo $policies | jq '."read-only"')"
   readWrite="$(echo $policies | jq '."read-write"')"
   [[ "$readOnly" != "{}" && "$readOnly" != "null" && "$readOnly" != "" ]]; because $? "when bucket and policies exist the result should include read-only policy"