From 0479682221990ccd193f319b129f0c0de97d9602 Mon Sep 17 00:00:00 2001 From: e-mhui Date: Sat, 11 Jan 2025 21:16:55 +0800 Subject: [PATCH 1/7] [Hotfix][Connector-V2][SFTP] Add quote to sftp file names with wildcard characters before get input stream --- .../file/sftp/system/SFTPFileSystem.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java index 83fccdeb3c4..80ee5459554 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java @@ -40,6 +40,7 @@ import java.io.OutputStream; import java.net.URI; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Vector; @@ -154,6 +155,29 @@ private boolean exists(ChannelSftp channel, Path file) throws IOException { } } + private String quote(String path) { + byte[] _path = path.getBytes(StandardCharsets.UTF_8); + int count = 0; + for (int i = 0; i < _path.length; i++) { + byte b = _path[i]; + if (b == '\\' || b == '?' || b == '*') { + count++; + } + } + if (count == 0) { + return path; + } + byte[] _path2 = new byte[_path.length + count]; + for (int i = 0, j = 0; i < _path.length; i++) { + byte b = _path[i]; + if (b == '\\' || b == '?' || b == '*') { + _path2[j++] = '\\'; + } + _path2[j++] = b; + } + return new String(_path2, 0, _path2.length, StandardCharsets.UTF_8); + } + /** * Convenience method, so that we don't open a new connection when using this method from within * another method. Otherwise every API invocation incurs the overhead of opening/closing a TCP @@ -466,7 +490,7 @@ public FSDataInputStream open(Path f, int bufferSize) throws IOException { // the path could be a symbolic link, so get the real path absolute = new Path("/", channel.realpath(absolute.toUri().getPath())); - is = channel.get(absolute.toUri().getPath()); + is = channel.get(quote(absolute.toUri().getPath())); } catch (SftpException e) { throw new IOException(e); } From e50446031be3044864395d353446c9836c7a33c3 Mon Sep 17 00:00:00 2001 From: e-mhui Date: Sun, 12 Jan 2025 14:40:49 +0800 Subject: [PATCH 2/7] [Hotfix][Connector-V2][SFTP] Add quote to sftp file names with wildcard characters before get input stream --- .../connectors/seatunnel/file/sftp/system/SFTPFileSystem.java | 1 + 1 file changed, 1 insertion(+) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java index 80ee5459554..f2f3f4fd480 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java @@ -175,6 +175,7 @@ private String quote(String path) { } _path2[j++] = b; } + return new String(_path2, 0, _path2.length, StandardCharsets.UTF_8); } From ae01c8abbf50334c22a8d7b2beec9a92c8327da4 Mon Sep 17 00:00:00 2001 From: e-mhui Date: Sun, 12 Jan 2025 17:18:42 +0800 Subject: [PATCH 3/7] Add UT --- .../e2e/connector/file/fstp/SftpFileIT.java | 12 ++ .../src/test/resources/wildcard/e*e.txt | 5 + .../src/test/resources/wildcard/e2e.txt | 5 + ...ile_text_wildcard_character_to_assert.conf | 117 ++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e*e.txt create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e2e.txt create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/sftp_file_text_wildcard_character_to_assert.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java index 2ac185aabbd..73f71189f62 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java @@ -113,6 +113,15 @@ public void startUp() throws Exception { "/home/seatunnel/tmp/seatunnel/read/xml/name=tyrantlucifer/hobby=coding/e2e.xml", sftpContainer); + ContainerUtil.copyFileIntoContainers( + "/wildcard/e*e.txt", + "/home/seatunnel/tmp/seatunnel/read/wildcard/e*e.txt", + sftpContainer); + + ContainerUtil.copyFileIntoContainers( + "/wildcard/e2e.txt", + "/home/seatunnel/tmp/seatunnel/read/wildcard/e2e.txt", + sftpContainer); sftpContainer.execInContainer("sh", "-c", "chown -R seatunnel /home/seatunnel/tmp/"); } @@ -147,6 +156,9 @@ public void testSftpFileReadAndWrite(TestContainer container) // test read sftp xml file helper.execute("/xml/sftp_file_xml_to_assert.conf"); // test sftp source support multipleTable + // test read file wit wildcard character, should match tmp/seatunnel/read/wildcard/e*e.txt + // and tmp/seatunnel/read/wildcard/e2e.txt + helper.execute("/wildcard/sftp_file_text_wildcard_character_to_assert.conf"); String homePath = "/home/seatunnel"; String sink01 = "/tmp/multipleSource/seatunnel/json/fake01"; String sink02 = "/tmp/multipleSource/seatunnel/json/fake02"; diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e*e.txt b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e*e.txt new file mode 100644 index 00000000000..9871cd85eb6 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e*e.txt @@ -0,0 +1,5 @@ +uDDrwsQQYONTNeUBIOnLAgunvDqLBObroRzdEdvDgRmgaeFyFH5456857591576298739157764687713794636442057612252MTDnafalse3313846190943192276641872220071936002.4798444E389.52375328387482E307vcIGF2023-06-0776258155390368615610.7646252373186602912023-05-08 16:08:51ipToEdierOAbwQfQzObWqiRhjkWYaMKdCbjurhstsWrAVlRyyR2905930362869031292782506910815576701385108050hArFutrue12631169122166306155952414159791708165.949173E372.1775762383875058E307kMlgO2023-05-2027214280267865241887.6424416000104182532023-10-20 03:49:02 +QIpzzZNFkLwARZDSdwdBzkegCdIRVYJnuXgxNXytAJxxaTzmDF16603816781145850255103997497062535321459349811xaTOktrue5327578191749099325840234439082792961.955231E381.5072154481920294E308GDWOu2023-05-0581449039533149712064.4515003874168475032023-07-06 22:34:11sfgxhqvOLzjdTSNcNaWfEnZqvQraSSuMPazCGhPmSrGuxggqGh111449466287130860562118177510004750271267350957FDhTstrue96247293946402921952995131535667203.3240283E384.473485404447698E307YFdwf2023-02-0429456519357128996647.9939318900994572132023-01-12 02:29:58 +xVJPgVlosBlTYSkmJCqKHMXzbZkNQKInuVMZeYGhsmzUmcLyPx137745493211075991209783701051546835517166168384qcYaifalse8318050110096656524405690917018449922.9617934E371.8901064340036343E307jaKMq2023-05-1275317114043170470995.9654034735914367862023-05-18 08:09:22raGGBnHsNwMZKemkFErUbedNjSllNcKOVUGdTpXcHGSVphHsNE86377304018502081846122308810391870441519757437JCRZStrue1829974183977114228752256792969205767.9090967E371.6286963710372255E308NBHUB2023-05-0732934086493941743464.6503746053883129532023-05-06 04:35:55 +dBgFeTKkCfnxCljyGfNEurEzCVgwpsHgmcOfYXiQHxeeQNjQuq1961913761867016982512369059615238191571813320BTfhbfalse652666522281866957533025299230722.1456136E381.2398422714159417E308YOiwg2023-10-2433001899362876139955.7235198795513055732023-06-23 13:46:46jsvmHLHlXCGFKwuqlTwAjdMckElrmqgBWvOuuKuWxcinFZWSky19959088245502706421265289671411088181469730839vUyULtrue952655754382886132164227350822215681.9033253E381.0966562906060974E308XFeKf2023-09-1731084757529957096723.2394423349193989032023-06-15 17:04:50 +obtYzIHOTKsABVtirEKEMYUYobsYlDJcFbpQUYvGxCcKlnswEG8096984004544201585383739017658796661353001394xchcntrue853141253976762312923177914159380482.8480754E381.055208146200822E308MSkTD2023-11-2420361788179232141281.9718823433892185262023-10-25 11:47:50gdCWZMGESyarjQPopBhDwKnOyDvaUDgQOEDRCmfUAagfnDDPqV8473436731118772451890654127233667151574025969ewJzLtrue6321769209768782446484076920790579202.7134378E381.1883616449174808E308STvOu2023-10-0821793351767634029460.2897683013563753232023-08-12 23:57:38 \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e2e.txt b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e2e.txt new file mode 100644 index 00000000000..9871cd85eb6 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e2e.txt @@ -0,0 +1,5 @@ +uDDrwsQQYONTNeUBIOnLAgunvDqLBObroRzdEdvDgRmgaeFyFH5456857591576298739157764687713794636442057612252MTDnafalse3313846190943192276641872220071936002.4798444E389.52375328387482E307vcIGF2023-06-0776258155390368615610.7646252373186602912023-05-08 16:08:51ipToEdierOAbwQfQzObWqiRhjkWYaMKdCbjurhstsWrAVlRyyR2905930362869031292782506910815576701385108050hArFutrue12631169122166306155952414159791708165.949173E372.1775762383875058E307kMlgO2023-05-2027214280267865241887.6424416000104182532023-10-20 03:49:02 +QIpzzZNFkLwARZDSdwdBzkegCdIRVYJnuXgxNXytAJxxaTzmDF16603816781145850255103997497062535321459349811xaTOktrue5327578191749099325840234439082792961.955231E381.5072154481920294E308GDWOu2023-05-0581449039533149712064.4515003874168475032023-07-06 22:34:11sfgxhqvOLzjdTSNcNaWfEnZqvQraSSuMPazCGhPmSrGuxggqGh111449466287130860562118177510004750271267350957FDhTstrue96247293946402921952995131535667203.3240283E384.473485404447698E307YFdwf2023-02-0429456519357128996647.9939318900994572132023-01-12 02:29:58 +xVJPgVlosBlTYSkmJCqKHMXzbZkNQKInuVMZeYGhsmzUmcLyPx137745493211075991209783701051546835517166168384qcYaifalse8318050110096656524405690917018449922.9617934E371.8901064340036343E307jaKMq2023-05-1275317114043170470995.9654034735914367862023-05-18 08:09:22raGGBnHsNwMZKemkFErUbedNjSllNcKOVUGdTpXcHGSVphHsNE86377304018502081846122308810391870441519757437JCRZStrue1829974183977114228752256792969205767.9090967E371.6286963710372255E308NBHUB2023-05-0732934086493941743464.6503746053883129532023-05-06 04:35:55 +dBgFeTKkCfnxCljyGfNEurEzCVgwpsHgmcOfYXiQHxeeQNjQuq1961913761867016982512369059615238191571813320BTfhbfalse652666522281866957533025299230722.1456136E381.2398422714159417E308YOiwg2023-10-2433001899362876139955.7235198795513055732023-06-23 13:46:46jsvmHLHlXCGFKwuqlTwAjdMckElrmqgBWvOuuKuWxcinFZWSky19959088245502706421265289671411088181469730839vUyULtrue952655754382886132164227350822215681.9033253E381.0966562906060974E308XFeKf2023-09-1731084757529957096723.2394423349193989032023-06-15 17:04:50 +obtYzIHOTKsABVtirEKEMYUYobsYlDJcFbpQUYvGxCcKlnswEG8096984004544201585383739017658796661353001394xchcntrue853141253976762312923177914159380482.8480754E381.055208146200822E308MSkTD2023-11-2420361788179232141281.9718823433892185262023-10-25 11:47:50gdCWZMGESyarjQPopBhDwKnOyDvaUDgQOEDRCmfUAagfnDDPqV8473436731118772451890654127233667151574025969ewJzLtrue6321769209768782446484076920790579202.7134378E381.1883616449174808E308STvOu2023-10-0821793351767634029460.2897683013563753232023-08-12 23:57:38 \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/sftp_file_text_wildcard_character_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/sftp_file_text_wildcard_character_to_assert.conf new file mode 100644 index 00000000000..3e24b131975 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/sftp_file_text_wildcard_character_to_assert.conf @@ -0,0 +1,117 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + SftpFile { + host = "sftp" + port = 22 + user = seatunnel + password = pass + path = "tmp/seatunnel/read/wildcard/" + file_format_type = "text" + plugin_output = "sftp" + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + c_row = { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_bytes = bytes + c_date = date + c_decimal = "decimal(38, 18)" + c_timestamp = timestamp + } + } + } + } +} + +sink { + Assert { + plugin_input = "sftp" + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 10 + } + ], + field_rules = [ + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_double + field_type = double + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} \ No newline at end of file From 8778743912c8b2a0947e946262d4efad13c08f69 Mon Sep 17 00:00:00 2001 From: e-mhui Date: Sun, 12 Jan 2025 17:22:13 +0800 Subject: [PATCH 4/7] [Hotfix][Connector-V2][SFTP] Add quote to sftp file names with wildcard characters before get input stream --- .../connectors/seatunnel/file/sftp/system/SFTPFileSystem.java | 1 - 1 file changed, 1 deletion(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java index f2f3f4fd480..80ee5459554 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sftp/system/SFTPFileSystem.java @@ -175,7 +175,6 @@ private String quote(String path) { } _path2[j++] = b; } - return new String(_path2, 0, _path2.length, StandardCharsets.UTF_8); } From f897337dc3fa034b2a2c8a92f3bfb2db86a7d57a Mon Sep 17 00:00:00 2001 From: emhui Date: Sun, 12 Jan 2025 20:11:28 +0800 Subject: [PATCH 5/7] Add UT --- .../e2e/connector/file/fstp/SftpFileIT.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java index 73f71189f62..cd1bed5283d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java @@ -31,6 +31,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; import org.testcontainers.containers.GenericContainer; import org.testcontainers.lifecycle.Startables; import org.testcontainers.shaded.com.github.dockerjava.core.command.ExecStartResultCallback; @@ -156,9 +158,6 @@ public void testSftpFileReadAndWrite(TestContainer container) // test read sftp xml file helper.execute("/xml/sftp_file_xml_to_assert.conf"); // test sftp source support multipleTable - // test read file wit wildcard character, should match tmp/seatunnel/read/wildcard/e*e.txt - // and tmp/seatunnel/read/wildcard/e2e.txt - helper.execute("/wildcard/sftp_file_text_wildcard_character_to_assert.conf"); String homePath = "/home/seatunnel"; String sink01 = "/tmp/multipleSource/seatunnel/json/fake01"; String sink02 = "/tmp/multipleSource/seatunnel/json/fake02"; @@ -202,6 +201,16 @@ public void testMultipleTableAndSaveMode(TestContainer container) Assertions.assertEquals(getFileListFromContainer(homePath + path4).size(), 2); } + @TestTemplate + @DisabledOnOs(OS.WINDOWS) + public void testSftpFileWithWildcardRead(TestContainer container) + throws IOException, InterruptedException { + TestHelper helper = new TestHelper(container); + // test read file wit wildcard character, should match tmp/seatunnel/read/wildcard/e*e.txt + // and tmp/seatunnel/read/wildcard/e2e.txt + helper.execute("/wildcard/sftp_file_text_wildcard_character_to_assert.conf"); + } + @SneakyThrows private List getFileListFromContainer(String path) { String command = "ls -1 " + path; From a0f55052b8fad367d1a5bd4bcc79cb2fbde9d697 Mon Sep 17 00:00:00 2001 From: emhui Date: Sun, 12 Jan 2025 20:53:08 +0800 Subject: [PATCH 6/7] Add UT --- .../seatunnel/e2e/connector/file/fstp/SftpFileIT.java | 6 ++++-- .../sftp_file_text_wildcard_character_to_assert.conf | 0 .../src/test/resources/wildcard/e*e.txt | 5 ----- .../src/test/resources/wildcard/e2e.txt | 5 ----- 4 files changed, 4 insertions(+), 12 deletions(-) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/{wildcard => text}/sftp_file_text_wildcard_character_to_assert.conf (100%) delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e*e.txt delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e2e.txt diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java index cd1bed5283d..f8999907dc8 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java @@ -115,8 +115,10 @@ public void startUp() throws Exception { "/home/seatunnel/tmp/seatunnel/read/xml/name=tyrantlucifer/hobby=coding/e2e.xml", sftpContainer); + // Windows does not support files with wildcard characters. We can rename `e2e.txt` to + // `e*e.txt` when copying to a container ContainerUtil.copyFileIntoContainers( - "/wildcard/e*e.txt", + "/wildcard/e2e.txt", "/home/seatunnel/tmp/seatunnel/read/wildcard/e*e.txt", sftpContainer); @@ -208,7 +210,7 @@ public void testSftpFileWithWildcardRead(TestContainer container) TestHelper helper = new TestHelper(container); // test read file wit wildcard character, should match tmp/seatunnel/read/wildcard/e*e.txt // and tmp/seatunnel/read/wildcard/e2e.txt - helper.execute("/wildcard/sftp_file_text_wildcard_character_to_assert.conf"); + helper.execute("/text/sftp_file_text_wildcard_character_to_assert.conf"); } @SneakyThrows diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/sftp_file_text_wildcard_character_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_wildcard_character_to_assert.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/sftp_file_text_wildcard_character_to_assert.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/text/sftp_file_text_wildcard_character_to_assert.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e*e.txt b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e*e.txt deleted file mode 100644 index 9871cd85eb6..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e*e.txt +++ /dev/null @@ -1,5 +0,0 @@ -uDDrwsQQYONTNeUBIOnLAgunvDqLBObroRzdEdvDgRmgaeFyFH5456857591576298739157764687713794636442057612252MTDnafalse3313846190943192276641872220071936002.4798444E389.52375328387482E307vcIGF2023-06-0776258155390368615610.7646252373186602912023-05-08 16:08:51ipToEdierOAbwQfQzObWqiRhjkWYaMKdCbjurhstsWrAVlRyyR2905930362869031292782506910815576701385108050hArFutrue12631169122166306155952414159791708165.949173E372.1775762383875058E307kMlgO2023-05-2027214280267865241887.6424416000104182532023-10-20 03:49:02 -QIpzzZNFkLwARZDSdwdBzkegCdIRVYJnuXgxNXytAJxxaTzmDF16603816781145850255103997497062535321459349811xaTOktrue5327578191749099325840234439082792961.955231E381.5072154481920294E308GDWOu2023-05-0581449039533149712064.4515003874168475032023-07-06 22:34:11sfgxhqvOLzjdTSNcNaWfEnZqvQraSSuMPazCGhPmSrGuxggqGh111449466287130860562118177510004750271267350957FDhTstrue96247293946402921952995131535667203.3240283E384.473485404447698E307YFdwf2023-02-0429456519357128996647.9939318900994572132023-01-12 02:29:58 -xVJPgVlosBlTYSkmJCqKHMXzbZkNQKInuVMZeYGhsmzUmcLyPx137745493211075991209783701051546835517166168384qcYaifalse8318050110096656524405690917018449922.9617934E371.8901064340036343E307jaKMq2023-05-1275317114043170470995.9654034735914367862023-05-18 08:09:22raGGBnHsNwMZKemkFErUbedNjSllNcKOVUGdTpXcHGSVphHsNE86377304018502081846122308810391870441519757437JCRZStrue1829974183977114228752256792969205767.9090967E371.6286963710372255E308NBHUB2023-05-0732934086493941743464.6503746053883129532023-05-06 04:35:55 -dBgFeTKkCfnxCljyGfNEurEzCVgwpsHgmcOfYXiQHxeeQNjQuq1961913761867016982512369059615238191571813320BTfhbfalse652666522281866957533025299230722.1456136E381.2398422714159417E308YOiwg2023-10-2433001899362876139955.7235198795513055732023-06-23 13:46:46jsvmHLHlXCGFKwuqlTwAjdMckElrmqgBWvOuuKuWxcinFZWSky19959088245502706421265289671411088181469730839vUyULtrue952655754382886132164227350822215681.9033253E381.0966562906060974E308XFeKf2023-09-1731084757529957096723.2394423349193989032023-06-15 17:04:50 -obtYzIHOTKsABVtirEKEMYUYobsYlDJcFbpQUYvGxCcKlnswEG8096984004544201585383739017658796661353001394xchcntrue853141253976762312923177914159380482.8480754E381.055208146200822E308MSkTD2023-11-2420361788179232141281.9718823433892185262023-10-25 11:47:50gdCWZMGESyarjQPopBhDwKnOyDvaUDgQOEDRCmfUAagfnDDPqV8473436731118772451890654127233667151574025969ewJzLtrue6321769209768782446484076920790579202.7134378E381.1883616449174808E308STvOu2023-10-0821793351767634029460.2897683013563753232023-08-12 23:57:38 \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e2e.txt b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e2e.txt deleted file mode 100644 index 9871cd85eb6..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/resources/wildcard/e2e.txt +++ /dev/null @@ -1,5 +0,0 @@ -uDDrwsQQYONTNeUBIOnLAgunvDqLBObroRzdEdvDgRmgaeFyFH5456857591576298739157764687713794636442057612252MTDnafalse3313846190943192276641872220071936002.4798444E389.52375328387482E307vcIGF2023-06-0776258155390368615610.7646252373186602912023-05-08 16:08:51ipToEdierOAbwQfQzObWqiRhjkWYaMKdCbjurhstsWrAVlRyyR2905930362869031292782506910815576701385108050hArFutrue12631169122166306155952414159791708165.949173E372.1775762383875058E307kMlgO2023-05-2027214280267865241887.6424416000104182532023-10-20 03:49:02 -QIpzzZNFkLwARZDSdwdBzkegCdIRVYJnuXgxNXytAJxxaTzmDF16603816781145850255103997497062535321459349811xaTOktrue5327578191749099325840234439082792961.955231E381.5072154481920294E308GDWOu2023-05-0581449039533149712064.4515003874168475032023-07-06 22:34:11sfgxhqvOLzjdTSNcNaWfEnZqvQraSSuMPazCGhPmSrGuxggqGh111449466287130860562118177510004750271267350957FDhTstrue96247293946402921952995131535667203.3240283E384.473485404447698E307YFdwf2023-02-0429456519357128996647.9939318900994572132023-01-12 02:29:58 -xVJPgVlosBlTYSkmJCqKHMXzbZkNQKInuVMZeYGhsmzUmcLyPx137745493211075991209783701051546835517166168384qcYaifalse8318050110096656524405690917018449922.9617934E371.8901064340036343E307jaKMq2023-05-1275317114043170470995.9654034735914367862023-05-18 08:09:22raGGBnHsNwMZKemkFErUbedNjSllNcKOVUGdTpXcHGSVphHsNE86377304018502081846122308810391870441519757437JCRZStrue1829974183977114228752256792969205767.9090967E371.6286963710372255E308NBHUB2023-05-0732934086493941743464.6503746053883129532023-05-06 04:35:55 -dBgFeTKkCfnxCljyGfNEurEzCVgwpsHgmcOfYXiQHxeeQNjQuq1961913761867016982512369059615238191571813320BTfhbfalse652666522281866957533025299230722.1456136E381.2398422714159417E308YOiwg2023-10-2433001899362876139955.7235198795513055732023-06-23 13:46:46jsvmHLHlXCGFKwuqlTwAjdMckElrmqgBWvOuuKuWxcinFZWSky19959088245502706421265289671411088181469730839vUyULtrue952655754382886132164227350822215681.9033253E381.0966562906060974E308XFeKf2023-09-1731084757529957096723.2394423349193989032023-06-15 17:04:50 -obtYzIHOTKsABVtirEKEMYUYobsYlDJcFbpQUYvGxCcKlnswEG8096984004544201585383739017658796661353001394xchcntrue853141253976762312923177914159380482.8480754E381.055208146200822E308MSkTD2023-11-2420361788179232141281.9718823433892185262023-10-25 11:47:50gdCWZMGESyarjQPopBhDwKnOyDvaUDgQOEDRCmfUAagfnDDPqV8473436731118772451890654127233667151574025969ewJzLtrue6321769209768782446484076920790579202.7134378E381.1883616449174808E308STvOu2023-10-0821793351767634029460.2897683013563753232023-08-12 23:57:38 \ No newline at end of file From 7136e9c6dd36de330e3e89d3159febfcb759ec5b Mon Sep 17 00:00:00 2001 From: e-mhui Date: Mon, 13 Jan 2025 10:25:11 +0800 Subject: [PATCH 7/7] Add missed files --- .../e2e/connector/file/fstp/SftpFileIT.java | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java index f8999907dc8..235f39ae38d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-sftp-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/fstp/SftpFileIT.java @@ -31,8 +31,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.api.condition.DisabledOnOs; -import org.junit.jupiter.api.condition.OS; import org.testcontainers.containers.GenericContainer; import org.testcontainers.lifecycle.Startables; import org.testcontainers.shaded.com.github.dockerjava.core.command.ExecStartResultCallback; @@ -118,12 +116,12 @@ public void startUp() throws Exception { // Windows does not support files with wildcard characters. We can rename `e2e.txt` to // `e*e.txt` when copying to a container ContainerUtil.copyFileIntoContainers( - "/wildcard/e2e.txt", + "/text/e2e.txt", "/home/seatunnel/tmp/seatunnel/read/wildcard/e*e.txt", sftpContainer); ContainerUtil.copyFileIntoContainers( - "/wildcard/e2e.txt", + "/text/e2e.txt", "/home/seatunnel/tmp/seatunnel/read/wildcard/e2e.txt", sftpContainer); sftpContainer.execInContainer("sh", "-c", "chown -R seatunnel /home/seatunnel/tmp/"); @@ -151,6 +149,9 @@ public void testSftpFileReadAndWrite(TestContainer container) helper.execute("/text/sftp_file_text_projection_to_assert.conf"); // test read sftp zip text file helper.execute("/text/sftp_file_zip_text_to_assert.conf"); + // test read file wit wildcard character, should match tmp/seatunnel/read/wildcard/e*e.txt + // and tmp/seatunnel/read/wildcard/e2e.txt + helper.execute("/text/sftp_file_text_wildcard_character_to_assert.conf"); // test write sftp json file helper.execute("/json/fake_to_sftp_file_json.conf"); // test read sftp json file @@ -203,16 +204,6 @@ public void testMultipleTableAndSaveMode(TestContainer container) Assertions.assertEquals(getFileListFromContainer(homePath + path4).size(), 2); } - @TestTemplate - @DisabledOnOs(OS.WINDOWS) - public void testSftpFileWithWildcardRead(TestContainer container) - throws IOException, InterruptedException { - TestHelper helper = new TestHelper(container); - // test read file wit wildcard character, should match tmp/seatunnel/read/wildcard/e*e.txt - // and tmp/seatunnel/read/wildcard/e2e.txt - helper.execute("/text/sftp_file_text_wildcard_character_to_assert.conf"); - } - @SneakyThrows private List getFileListFromContainer(String path) { String command = "ls -1 " + path;