From 1c510f5344fa92cb9577a00528897fd0c326e95f Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle <54168385+mgabelle@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:08:38 +0100 Subject: [PATCH] refactor: migrate auoCommit to dynamic props (#482) --- .../main/java/io/kestra/plugin/jdbc/as400/Query.java | 5 +++-- .../java/io/kestra/plugin/jdbc/clickhouse/Query.java | 5 +++-- .../main/java/io/kestra/plugin/jdbc/db2/Query.java | 3 ++- .../main/java/io/kestra/plugin/jdbc/duckdb/Query.java | 2 +- .../main/java/io/kestra/plugin/jdbc/mysql/Query.java | 2 +- .../main/java/io/kestra/plugin/jdbc/oracle/Query.java | 5 +++-- .../java/io/kestra/plugin/jdbc/postgresql/Query.java | 2 +- .../java/io/kestra/plugin/jdbc/redshift/Query.java | 3 ++- .../java/io/kestra/plugin/jdbc/snowflake/Query.java | 2 +- .../main/java/io/kestra/plugin/jdbc/sqlite/Query.java | 11 ++++++----- .../java/io/kestra/plugin/jdbc/sqlserver/Query.java | 5 +++-- .../main/java/io/kestra/plugin/jdbc/sybase/Query.java | 5 +++-- .../main/java/io/kestra/plugin/jdbc/trino/Query.java | 7 ++++--- .../java/io/kestra/plugin/jdbc/vectorwise/Query.java | 3 ++- .../java/io/kestra/plugin/jdbc/vertica/Query.java | 3 ++- .../java/io/kestra/plugin/jdbc/AbstractJdbcQuery.java | 2 +- .../io/kestra/plugin/jdbc/AutoCommitInterface.java | 5 ++--- 17 files changed, 40 insertions(+), 30 deletions(-) diff --git a/plugin-jdbc-as400/src/main/java/io/kestra/plugin/jdbc/as400/Query.java b/plugin-jdbc-as400/src/main/java/io/kestra/plugin/jdbc/as400/Query.java index 426712af..65728643 100644 --- a/plugin-jdbc-as400/src/main/java/io/kestra/plugin/jdbc/as400/Query.java +++ b/plugin-jdbc-as400/src/main/java/io/kestra/plugin/jdbc/as400/Query.java @@ -2,6 +2,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.jdbc.AbstractCellConverter; @@ -37,7 +38,7 @@ code = """ id: as400_query namespace: company.team - + tasks: - id: query type: io.kestra.plugin.jdbc.as400.Query @@ -51,7 +52,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc-clickhouse/src/main/java/io/kestra/plugin/jdbc/clickhouse/Query.java b/plugin-jdbc-clickhouse/src/main/java/io/kestra/plugin/jdbc/clickhouse/Query.java index 9db5f2da..cb8c78e4 100644 --- a/plugin-jdbc-clickhouse/src/main/java/io/kestra/plugin/jdbc/clickhouse/Query.java +++ b/plugin-jdbc-clickhouse/src/main/java/io/kestra/plugin/jdbc/clickhouse/Query.java @@ -1,5 +1,6 @@ package io.kestra.plugin.jdbc.clickhouse; +import io.kestra.core.models.property.Property; import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; @@ -34,7 +35,7 @@ code = """ id: clickhouse_query namespace: company.team - + tasks: - id: query type: io.kestra.plugin.jdbc.clickhouse.Query @@ -48,7 +49,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc-db2/src/main/java/io/kestra/plugin/jdbc/db2/Query.java b/plugin-jdbc-db2/src/main/java/io/kestra/plugin/jdbc/db2/Query.java index dfe02f76..c0fa0ec8 100644 --- a/plugin-jdbc-db2/src/main/java/io/kestra/plugin/jdbc/db2/Query.java +++ b/plugin-jdbc-db2/src/main/java/io/kestra/plugin/jdbc/db2/Query.java @@ -2,6 +2,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.jdbc.AbstractCellConverter; @@ -48,7 +49,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc-duckdb/src/main/java/io/kestra/plugin/jdbc/duckdb/Query.java b/plugin-jdbc-duckdb/src/main/java/io/kestra/plugin/jdbc/duckdb/Query.java index 8146035c..858b3ade 100644 --- a/plugin-jdbc-duckdb/src/main/java/io/kestra/plugin/jdbc/duckdb/Query.java +++ b/plugin-jdbc-duckdb/src/main/java/io/kestra/plugin/jdbc/duckdb/Query.java @@ -115,7 +115,7 @@ public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { private static final String DEFAULT_URL = "jdbc:duckdb:"; - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Schema( title = "Input files to be loaded from DuckDb.", diff --git a/plugin-jdbc-mysql/src/main/java/io/kestra/plugin/jdbc/mysql/Query.java b/plugin-jdbc-mysql/src/main/java/io/kestra/plugin/jdbc/mysql/Query.java index af760a2b..65d8a11a 100644 --- a/plugin-jdbc-mysql/src/main/java/io/kestra/plugin/jdbc/mysql/Query.java +++ b/plugin-jdbc-mysql/src/main/java/io/kestra/plugin/jdbc/mysql/Query.java @@ -82,7 +82,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Schema( title = "Add input file to be loaded with `LOAD DATA LOCAL`.", diff --git a/plugin-jdbc-oracle/src/main/java/io/kestra/plugin/jdbc/oracle/Query.java b/plugin-jdbc-oracle/src/main/java/io/kestra/plugin/jdbc/oracle/Query.java index 8dc4125b..e24de4cd 100644 --- a/plugin-jdbc-oracle/src/main/java/io/kestra/plugin/jdbc/oracle/Query.java +++ b/plugin-jdbc-oracle/src/main/java/io/kestra/plugin/jdbc/oracle/Query.java @@ -1,5 +1,6 @@ package io.kestra.plugin.jdbc.oracle; +import io.kestra.core.models.property.Property; import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; @@ -42,7 +43,7 @@ password: oracle_password sql: select * from source fetchType: FETCH - + - id: generate_update type: io.kestra.plugin.jdbc.oracle.Query url: jdbc:oracle:thin:@localhost:49161:XE @@ -54,7 +55,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc-postgres/src/main/java/io/kestra/plugin/jdbc/postgresql/Query.java b/plugin-jdbc-postgres/src/main/java/io/kestra/plugin/jdbc/postgresql/Query.java index ed91a919..00d62f4b 100644 --- a/plugin-jdbc-postgres/src/main/java/io/kestra/plugin/jdbc/postgresql/Query.java +++ b/plugin-jdbc-postgres/src/main/java/io/kestra/plugin/jdbc/postgresql/Query.java @@ -56,7 +56,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, PostgresConnectionInterface, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Builder.Default protected Property ssl = Property.of(false); protected Property sslMode; diff --git a/plugin-jdbc-redshift/src/main/java/io/kestra/plugin/jdbc/redshift/Query.java b/plugin-jdbc-redshift/src/main/java/io/kestra/plugin/jdbc/redshift/Query.java index ac7de9de..6f5bfb60 100644 --- a/plugin-jdbc-redshift/src/main/java/io/kestra/plugin/jdbc/redshift/Query.java +++ b/plugin-jdbc-redshift/src/main/java/io/kestra/plugin/jdbc/redshift/Query.java @@ -1,5 +1,6 @@ package io.kestra.plugin.jdbc.redshift; +import io.kestra.core.models.property.Property; import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; @@ -47,7 +48,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Query.java b/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Query.java index a7d24bf0..74f87b1e 100644 --- a/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Query.java +++ b/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Query.java @@ -57,7 +57,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, SnowflakeInterface, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); private Property privateKey; private Property privateKeyFile; diff --git a/plugin-jdbc-sqlite/src/main/java/io/kestra/plugin/jdbc/sqlite/Query.java b/plugin-jdbc-sqlite/src/main/java/io/kestra/plugin/jdbc/sqlite/Query.java index e5f7d1ee..aab6b65d 100644 --- a/plugin-jdbc-sqlite/src/main/java/io/kestra/plugin/jdbc/sqlite/Query.java +++ b/plugin-jdbc-sqlite/src/main/java/io/kestra/plugin/jdbc/sqlite/Query.java @@ -3,6 +3,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.models.tasks.runners.PluginUtilsService; import io.kestra.core.runners.RunContext; @@ -39,14 +40,14 @@ code = """ id: sqlite_query namespace: company.team - + tasks: - id: update type: io.kestra.plugin.jdbc.sqlite.Query url: jdbc:sqlite:myfile.db sql: select concert_id, available, a, b, c, d, play_time, library_record, floatn_test, double_test, real_test, numeric_test, date_type, time_type, timez_type, timestamp_type, timestampz_type, interval_type, pay_by_quarter, schedule, json_type, blob_type from pgsql_types fetchType: FETCH - + - id: use_fetched_data type: io.kestra.plugin.jdbc.sqlite.Query url: jdbc:sqlite:myfile.db @@ -59,7 +60,7 @@ code = """ id: sqlite_query_using_file namespace: company.team - + tasks: - id: update type: io.kestra.plugin.jdbc.sqlite.Query @@ -67,7 +68,7 @@ sqliteFile: {{ outputs.get.outputFiles['myfile.sqlite'] }} sql: select * from pgsql_types fetchType: FETCH - + - id: use_fetched_data type: io.kestra.plugin.jdbc.sqlite.Query url: jdbc:sqlite:myfile.db @@ -78,7 +79,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Schema( title = "Add sqlite file.", diff --git a/plugin-jdbc-sqlserver/src/main/java/io/kestra/plugin/jdbc/sqlserver/Query.java b/plugin-jdbc-sqlserver/src/main/java/io/kestra/plugin/jdbc/sqlserver/Query.java index dc09b68a..1fde19f0 100644 --- a/plugin-jdbc-sqlserver/src/main/java/io/kestra/plugin/jdbc/sqlserver/Query.java +++ b/plugin-jdbc-sqlserver/src/main/java/io/kestra/plugin/jdbc/sqlserver/Query.java @@ -1,5 +1,6 @@ package io.kestra.plugin.jdbc.sqlserver; +import io.kestra.core.models.property.Property; import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; @@ -42,7 +43,7 @@ password: sql_server_password sql: select * from source fetchType: FETCH - + - id: generate_update type: io.kestra.plugin.jdbc.sqlserver.Query url: jdbc:sqlserver://localhost:41433;trustServerCertificate=true @@ -54,7 +55,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc-sybase/src/main/java/io/kestra/plugin/jdbc/sybase/Query.java b/plugin-jdbc-sybase/src/main/java/io/kestra/plugin/jdbc/sybase/Query.java index 1eed6c4e..797cf670 100644 --- a/plugin-jdbc-sybase/src/main/java/io/kestra/plugin/jdbc/sybase/Query.java +++ b/plugin-jdbc-sybase/src/main/java/io/kestra/plugin/jdbc/sybase/Query.java @@ -2,6 +2,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.jdbc.AbstractCellConverter; @@ -34,7 +35,7 @@ full = true, code = """ id: sybase_query - namespace: company.team + namespace: company.team tasks: - id: query @@ -49,7 +50,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc-trino/src/main/java/io/kestra/plugin/jdbc/trino/Query.java b/plugin-jdbc-trino/src/main/java/io/kestra/plugin/jdbc/trino/Query.java index 7cf26532..3da1b81a 100644 --- a/plugin-jdbc-trino/src/main/java/io/kestra/plugin/jdbc/trino/Query.java +++ b/plugin-jdbc-trino/src/main/java/io/kestra/plugin/jdbc/trino/Query.java @@ -1,5 +1,6 @@ package io.kestra.plugin.jdbc.trino; +import io.kestra.core.models.property.Property; import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; @@ -33,7 +34,7 @@ code = """ id: trino_query namespace: company.team - + tasks: - id: analyze_orders type: io.kestra.plugin.jdbc.trino.Query @@ -47,7 +48,7 @@ select orderpriority as priority, sum(totalprice) as total order by orderpriority fetchType: FETCH fetchType: STORE - + - id: csv_report type: io.kestra.plugin.serdes.csv.IonToCsv from: "{{ outputs.analyze_orders.uri }}" @@ -56,7 +57,7 @@ select orderpriority as priority, sum(totalprice) as total } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc-vectorwise/src/main/java/io/kestra/plugin/jdbc/vectorwise/Query.java b/plugin-jdbc-vectorwise/src/main/java/io/kestra/plugin/jdbc/vectorwise/Query.java index 6fa97280..f7503847 100644 --- a/plugin-jdbc-vectorwise/src/main/java/io/kestra/plugin/jdbc/vectorwise/Query.java +++ b/plugin-jdbc-vectorwise/src/main/java/io/kestra/plugin/jdbc/vectorwise/Query.java @@ -1,5 +1,6 @@ package io.kestra.plugin.jdbc.vectorwise; +import io.kestra.core.models.property.Property; import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; @@ -47,7 +48,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc-vertica/src/main/java/io/kestra/plugin/jdbc/vertica/Query.java b/plugin-jdbc-vertica/src/main/java/io/kestra/plugin/jdbc/vertica/Query.java index 182c30ac..274ca21a 100644 --- a/plugin-jdbc-vertica/src/main/java/io/kestra/plugin/jdbc/vertica/Query.java +++ b/plugin-jdbc-vertica/src/main/java/io/kestra/plugin/jdbc/vertica/Query.java @@ -2,6 +2,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.jdbc.AbstractCellConverter; @@ -48,7 +49,7 @@ } ) public class Query extends AbstractJdbcQuery implements RunnableTask, AutoCommitInterface { - protected final Boolean autoCommit = true; + protected final Property autoCommit = Property.of(true); @Override protected AbstractCellConverter getCellConverter(ZoneId zoneId) { diff --git a/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AbstractJdbcQuery.java b/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AbstractJdbcQuery.java index 7d42e93e..6560717d 100644 --- a/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AbstractJdbcQuery.java +++ b/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AbstractJdbcQuery.java @@ -40,7 +40,7 @@ public AbstractJdbcBaseQuery.Output run(RunContext runContext) throws Exception if (this.renderFetchType(runContext).equals(FetchType.STORE)) { conn.setAutoCommit(false); } else { - conn.setAutoCommit(autoCommitClass.getAutoCommit()); + conn.setAutoCommit(runContext.render(autoCommitClass.getAutoCommit()).as(Boolean.class).orElse(true)); } } diff --git a/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AutoCommitInterface.java b/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AutoCommitInterface.java index fed16f2f..6075e56a 100644 --- a/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AutoCommitInterface.java +++ b/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AutoCommitInterface.java @@ -1,6 +1,6 @@ package io.kestra.plugin.jdbc; -import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.swagger.v3.oas.annotations.media.Schema; public interface AutoCommitInterface { @@ -12,6 +12,5 @@ public interface AutoCommitInterface { "or the method rollback. By default, new connections are in auto-commit mode except when you are using " + "`store` property in which case the auto-commit will be disabled." ) - @PluginProperty(dynamic = false) - Boolean getAutoCommit(); + Property getAutoCommit(); }