Hadoop and AWS sdk version upgrade

apache · Dec 31, 2024 · 7563aaa · 7563aaa
1 parent d80c957
commit 7563aaa
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 11 deletions.
diff --git a/pom.xml b/pom.xml
@@ -61,8 +61,9 @@
         <junit.version>5.9.0</junit.version>
         <lombok.version>1.18.30</lombok.version>
         <lombok-maven-plugin.version>1.18.20.0</lombok-maven-plugin.version>
-        <hadoop.version>3.4.0</hadoop.version>
+        <hadoop.version>3.4.1</hadoop.version>
         <hudi.version>0.14.0</hudi.version>
+        <aws.version>2.29.40</aws.version>
         <maven-source-plugin.version>3.3.1</maven-source-plugin.version>
         <maven-javadoc-plugin.version>3.8.0</maven-javadoc-plugin.version>
         <maven-gpg-plugin.version>3.2.4</maven-gpg-plugin.version>
@@ -371,19 +372,21 @@
                 <version>${hadoop.version}</version>
                 <scope>runtime</scope>
             </dependency>
-            <dependency>
-                <groupId>com.amazonaws</groupId>
-                <artifactId>aws-java-sdk-bundle</artifactId>
-                <version>1.12.328</version>
-                <scope>runtime</scope>
-            </dependency>
             <dependency>
                 <groupId>com.google.cloud.bigdataoss</groupId>
                 <artifactId>gcs-connector</artifactId>
                 <version>hadoop3-2.2.22</version>
                 <scope>runtime</scope>
             </dependency>
 
+            <!-- AWS -->
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>bundle</artifactId>
+                <version>${aws.version}</version>
+                <scope>runtime</scope>
+            </dependency>
+
             <!-- Protobuf -->
             <dependency>
                 <groupId>com.google.protobuf</groupId>

diff --git a/xtable-utilities/pom.xml b/xtable-utilities/pom.xml
@@ -98,8 +98,8 @@
             <artifactId>hadoop-aws</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-bundle</artifactId>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>bundle</artifactId>
         </dependency>
 
         <!-- Azure Dependencies -->

diff --git a/xtable-utilities/src/main/resources/xtable-hadoop-defaults.xml b/xtable-utilities/src/main/resources/xtable-hadoop-defaults.xml
@@ -55,15 +55,15 @@
   </property>
   <property>
     <name>fs.s3.aws.credentials.provider</name>
-    <value>com.amazonaws.auth.DefaultAWSCredentialsProviderChain</value>
+    <value>software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider</value>
   </property>
   <property>
     <name>fs.s3a.impl</name>
     <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
   </property>
   <property>
     <name>fs.s3a.aws.credentials.provider</name>
-    <value>com.amazonaws.auth.DefaultAWSCredentialsProviderChain</value>
+    <value>software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider</value>
   </property>
 
   <!-- Default file system for GCP scheme, gs:// -->