Integration wizard PoC (#1310)

Co-authored-by: alexeykozyurov <[email protected]>

gradle/libs.versions.toml

@@ -4,6 +4,7 @@ reactor-extra = '3.4.8'
 micrometer-registry-prometheus = '1.9.0'
 ingestion-contract-server = '0.1.22'
 oddrn-generator-java = '0.1.10'
+odd-integration-manifests = '0.0.2'
 apache-collections = '4.4'
 apache-lang = '3.12.0'
 r2dbc-spi = '1.0.0.RELEASE'
@@ -18,6 +19,7 @@ javax-annotation = '1.3.2'
 jackson-annotations = '2.13.2'
 jackson-datatype-jsr310 = '2.13.3'
 jackson-databind-nullable = '0.2.2'
+jackson-yaml = '2.13.0'
 jetbrains-annotations = '23.0.0'
 swagger-annotations = '1.5.23'
 springfox-core = '3.0.0'
@@ -57,6 +59,7 @@ reactor-extra = { module = 'io.projectreactor.addons:reactor-extra', version.ref
 micrometer-registry-prometheus = { module = 'io.micrometer:micrometer-registry-prometheus', version.ref = 'micrometer-registry-prometheus' }
 ingestion-contract-server = { module = 'org.opendatadiscovery:ingestion-contract-server', version.ref = 'ingestion-contract-server' }
 oddrn-generator-java = { module = 'org.opendatadiscovery:oddrn-generator-java', version.ref = 'oddrn-generator-java' }
+odd-integration-manifests = { module = 'org.opendatadiscovery:integration-manifests', version.ref = 'odd-integration-manifests' }
 apache-collections = { module = 'org.apache.commons:commons-collections4', version.ref = 'apache-collections' }
 apache-lang = { module = 'org.apache.commons:commons-lang3', version.ref = 'apache-lang' }
 r2dbc-spi = { module = 'io.r2dbc:r2dbc-spi', version.ref = 'r2dbc-spi' }
@@ -71,6 +74,7 @@ javax-annotation = { module = 'javax.annotation:javax.annotation-api', version.r
 jackson-datatype-jsr310 = { module = 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310', version.ref = 'jackson-datatype-jsr310' }
 jackson-databind-nullable = { module = 'org.openapitools:jackson-databind-nullable', version.ref = 'jackson-databind-nullable' }
 jackson-annotations = { module = 'com.fasterxml.jackson.core:jackson-annotations', version.ref = 'jackson-annotations' }
+jackson-yaml = { module = 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml', version.ref = 'jackson-yaml' }
 jetbrains-annotations = { module = 'org.jetbrains:annotations', version.ref = 'jetbrains-annotations' }
 swagger-annotations = { module = 'io.swagger:swagger-annotations', version.ref = 'swagger-annotations' }
 springfox-core = { module = 'io.springfox:springfox-core', version.ref = 'springfox-core' }
@@ -123,7 +127,8 @@ spring-redis-session = [
 ]
 internal = [
     'ingestion-contract-server',
-    'oddrn-generator-java'
+    'oddrn-generator-java',
+    'odd-integration-manifests'
 ]
 apache-commons = [
     'apache-collections',

odd-platform-api/build.gradle

@@ -31,6 +31,7 @@ dependencies {
     implementation libs.jackson.datatype.jsr310
     implementation libs.jackson.databind.nullable
     implementation libs.jackson.annotations
+    implementation libs.jackson.yaml
     implementation libs.jetbrains.annotations
     implementation libs.mapstruct
     implementation libs.slack.api

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/IntegrationController.java

@@ -0,0 +1,28 @@
+package org.opendatadiscovery.oddplatform.controller;
+
+import lombok.RequiredArgsConstructor;
+import org.opendatadiscovery.oddplatform.api.contract.api.IntegrationApi;
+import org.opendatadiscovery.oddplatform.api.contract.model.Integration;
+import org.opendatadiscovery.oddplatform.api.contract.model.IntegrationPreviewList;
+import org.opendatadiscovery.oddplatform.integration.service.IntegrationService;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.server.ServerWebExchange;
+import reactor.core.publisher.Mono;
+
+@RestController
+@RequiredArgsConstructor
+public class IntegrationController implements IntegrationApi {
+    private final IntegrationService integrationService;
+
+    @Override
+    public Mono<ResponseEntity<Integration>> getIntegration(final String integrationId,
+                                                            final ServerWebExchange exchange) {
+        return integrationService.get(integrationId).map(ResponseEntity::ok);
+    }
+
+    @Override
+    public Mono<ResponseEntity<IntegrationPreviewList>> getIntegrationPreviews(final ServerWebExchange exchange) {
+        return integrationService.listPreviews().map(ResponseEntity::ok);
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/IntegrationRegistry.java

@@ -0,0 +1,12 @@
+package org.opendatadiscovery.oddplatform.integration;
+
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationOverviewDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationPreviewDto;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+
+public interface IntegrationRegistry {
+    Mono<IntegrationOverviewDto> get(final String id);
+
+    Flux<IntegrationPreviewDto> list();
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/IntegrationRegistryFactory.java

@@ -0,0 +1,55 @@
+package org.opendatadiscovery.oddplatform.integration;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationOverviewDto;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+import org.springframework.core.io.support.ResourcePatternResolver;
+
+import static java.util.function.Function.identity;
+
+@UtilityClass
+@Slf4j
+public class IntegrationRegistryFactory {
+    private static final ResourcePatternResolver RESOURCE_PATTERN_RESOLVER = new PathMatchingResourcePatternResolver();
+    private static final String CLASSPATH_RESOURCE_LOCATION = "classpath*:META-INF/wizard/*.yaml";
+    private static final ObjectMapper YAML_OBJ_MAPPER = new ObjectMapper(new YAMLFactory());
+
+    public static IntegrationRegistry createResourceFilesIntegrationRegistry() {
+        final Map<String, IntegrationOverviewDto> registry = readManifests()
+            .stream()
+            .collect(Collectors.toMap(o -> o.integration().id(), identity()));
+
+        return new ResourceFilesIntegrationRegistry(registry);
+    }
+
+    private static List<IntegrationOverviewDto> readManifests() {
+        try {
+            return Arrays.stream(RESOURCE_PATTERN_RESOLVER.getResources(CLASSPATH_RESOURCE_LOCATION))
+                .filter(Resource::isReadable)
+                .map(IntegrationRegistryFactory::readManifest)
+                .toList();
+        } catch (final IOException e) {
+            throw new IllegalStateException("Couldn't read wizard manifests", e);
+        }
+    }
+
+    private static IntegrationOverviewDto readManifest(final Resource resource) {
+        log.debug("Reading wizard manifest file: {}", resource.getFilename());
+
+        try (final InputStream is = resource.getInputStream()) {
+            return YAML_OBJ_MAPPER.readValue(is, IntegrationOverviewDto.class);
+        } catch (final IOException e) {
+            throw new IllegalStateException("Couldn't read wizard manifest: %s".formatted(resource.getFilename()), e);
+        }
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/ResourceFilesIntegrationRegistry.java

@@ -0,0 +1,23 @@
+package org.opendatadiscovery.oddplatform.integration;
+
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationOverviewDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationPreviewDto;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+
+@RequiredArgsConstructor
+public class ResourceFilesIntegrationRegistry implements IntegrationRegistry {
+    private final Map<String, IntegrationOverviewDto> registry;
+
+    @Override
+    public Mono<IntegrationOverviewDto> get(final String id) {
+        return Mono.justOrEmpty(registry.get(id));
+    }
+
+    @Override
+    public Flux<IntegrationPreviewDto> list() {
+        return Flux.fromIterable(registry.values()).map(IntegrationOverviewDto::integration);
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/config/IntegrationConfiguration.java

@@ -0,0 +1,14 @@
+package org.opendatadiscovery.oddplatform.integration.config;
+
+import org.opendatadiscovery.oddplatform.integration.IntegrationRegistry;
+import org.opendatadiscovery.oddplatform.integration.IntegrationRegistryFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class IntegrationConfiguration {
+    @Bean
+    public IntegrationRegistry integrationRegistry() {
+        return IntegrationRegistryFactory.createResourceFilesIntegrationRegistry();
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/dto/IntegrationCodeSnippetArgumentDto.java

@@ -0,0 +1,6 @@
+package org.opendatadiscovery.oddplatform.integration.dto;
+
+public record IntegrationCodeSnippetArgumentDto(String parameter,
+                                                String name,
+                                                IntegrationCodeSnippetArgumentTypeEnum type) {
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/dto/IntegrationCodeSnippetArgumentTypeEnum.java

@@ -0,0 +1,8 @@
+package org.opendatadiscovery.oddplatform.integration.dto;
+
+public enum IntegrationCodeSnippetArgumentTypeEnum {
+    INTEGER,
+    STRING,
+    BOOLEAN,
+    FLOAT
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/dto/IntegrationCodeSnippetDto.java

@@ -0,0 +1,6 @@
+package org.opendatadiscovery.oddplatform.integration.dto;
+
+import java.util.List;
+
+public record IntegrationCodeSnippetDto(String template, List<IntegrationCodeSnippetArgumentDto> arguments) {
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/dto/IntegrationContentBlockDto.java

@@ -0,0 +1,6 @@
+package org.opendatadiscovery.oddplatform.integration.dto;
+
+import java.util.List;
+
+public record IntegrationContentBlockDto(String title, String content, List<IntegrationCodeSnippetDto> codeSnippets) {
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/dto/IntegrationOverviewDto.java

@@ -0,0 +1,10 @@
+package org.opendatadiscovery.oddplatform.integration.dto;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import java.util.List;
+import org.opendatadiscovery.oddplatform.integration.serde.IntegrationDeserializer;
+
+@JsonDeserialize(using = IntegrationDeserializer.class)
+public record IntegrationOverviewDto(IntegrationPreviewDto integration,
+                                     List<IntegrationContentBlockDto> contentBlocks) {
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/dto/IntegrationPreviewDto.java

@@ -0,0 +1,4 @@
+package org.opendatadiscovery.oddplatform.integration.dto;
+
+public record IntegrationPreviewDto(String id, String name, String description) {
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/mapper/IntegrationMapper.java

@@ -0,0 +1,45 @@
+package org.opendatadiscovery.oddplatform.integration.mapper;
+
+import java.util.List;
+import org.mapstruct.Mapper;
+import org.mapstruct.Mapping;
+import org.opendatadiscovery.oddplatform.api.contract.model.Integration;
+import org.opendatadiscovery.oddplatform.api.contract.model.IntegrationCodeSnippet;
+import org.opendatadiscovery.oddplatform.api.contract.model.IntegrationCodeSnippetArgument;
+import org.opendatadiscovery.oddplatform.api.contract.model.IntegrationCodeSnippetArgumentType;
+import org.opendatadiscovery.oddplatform.api.contract.model.IntegrationContentBlock;
+import org.opendatadiscovery.oddplatform.api.contract.model.IntegrationPreview;
+import org.opendatadiscovery.oddplatform.api.contract.model.IntegrationPreviewList;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationCodeSnippetArgumentDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationCodeSnippetArgumentTypeEnum;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationCodeSnippetDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationContentBlockDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationOverviewDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationPreviewDto;
+import org.opendatadiscovery.oddplatform.mapper.MapperConfig;
+
+@Mapper(config = MapperConfig.class)
+public interface IntegrationMapper {
+    @Mapping(target = "id", source = "integration.id")
+    @Mapping(target = "name", source = "integration.name")
+    @Mapping(target = "description", source = "integration.description")
+    @Mapping(target = "installed", constant = "false")
+    Integration map(final IntegrationOverviewDto dto);
+
+    @Mapping(target = "installed", constant = "false")
+    IntegrationPreview map(final IntegrationPreviewDto dto);
+
+    IntegrationContentBlock map(final IntegrationContentBlockDto dto);
+
+    IntegrationCodeSnippet map(final IntegrationCodeSnippetDto dto);
+
+    IntegrationCodeSnippetArgument map(final IntegrationCodeSnippetArgumentDto dto);
+
+    default IntegrationCodeSnippetArgumentType map(final IntegrationCodeSnippetArgumentTypeEnum dto) {
+        return IntegrationCodeSnippetArgumentType.fromValue(dto.name());
+    }
+
+    default IntegrationPreviewList map(final List<IntegrationPreviewDto> dtos) {
+        return new IntegrationPreviewList().items(dtos.stream().map(this::map).toList());
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/serde/IntegrationDeserializer.java

@@ -0,0 +1,75 @@
+package org.opendatadiscovery.oddplatform.integration.serde;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.lang3.ObjectUtils;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationCodeSnippetArgumentDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationCodeSnippetArgumentTypeEnum;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationCodeSnippetDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationContentBlockDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationOverviewDto;
+import org.opendatadiscovery.oddplatform.integration.dto.IntegrationPreviewDto;
+
+public class IntegrationDeserializer extends StdDeserializer<IntegrationOverviewDto> {
+    public IntegrationDeserializer() {
+        this(null);
+    }
+
+    protected IntegrationDeserializer(final Class<?> vc) {
+        super(vc);
+    }
+
+    @Override
+    public IntegrationOverviewDto deserialize(
+        final JsonParser p,
+        final DeserializationContext ctx
+    ) throws IOException {
+        final ObjectNode integrationNode = p.getCodec().readTree(p);
+
+        final String id = integrationNode.get("id").asText();
+        final String name = integrationNode.get("name").asText();
+        final String description = integrationNode.get("description").asText();
+
+        final IntegrationPreviewDto integrationPreviewDto = new IntegrationPreviewDto(id, name, description);
+        final List<IntegrationContentBlockDto> integrationContentBlockDtos = new ArrayList<>();
+
+        for (final JsonNode block : integrationNode.get("blocks")) {
+            final String blockTitle = block.get("title").asText();
+            final String blockContent = block.get("content").asText();
+            final JsonNode blockSnippets = block.get("snippets");
+
+            final List<IntegrationCodeSnippetDto> snippets = new ArrayList<>();
+
+            for (final JsonNode snippet : ObjectUtils.defaultIfNull(blockSnippets, List.<JsonNode>of())) {
+                final String snippetTemplate = snippet.get("template").asText();
+
+                final List<IntegrationCodeSnippetArgumentDto> arguments = new ArrayList<>();
+                final Iterable<JsonNode> snippetArgs = ObjectUtils.defaultIfNull(snippet.get("arguments"), List.of());
+
+                for (final JsonNode argument : snippetArgs) {
+                    final String argParameter = argument.get("parameter").asText();
+                    final String argName = argument.get("name").asText();
+                    final String type = argument.get("type").asText();
+
+                    arguments.add(new IntegrationCodeSnippetArgumentDto(
+                        argParameter,
+                        argName,
+                        IntegrationCodeSnippetArgumentTypeEnum.valueOf(type.toUpperCase())
+                    ));
+                }
+
+                snippets.add(new IntegrationCodeSnippetDto(snippetTemplate, arguments));
+            }
+
+            integrationContentBlockDtos.add(new IntegrationContentBlockDto(blockTitle, blockContent, snippets));
+        }
+
+        return new IntegrationOverviewDto(integrationPreviewDto, integrationContentBlockDtos);
+    }
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/service/IntegrationService.java

@@ -0,0 +1,11 @@
+package org.opendatadiscovery.oddplatform.integration.service;
+
+import org.opendatadiscovery.oddplatform.api.contract.model.Integration;
+import org.opendatadiscovery.oddplatform.api.contract.model.IntegrationPreviewList;
+import reactor.core.publisher.Mono;
+
+public interface IntegrationService {
+    Mono<Integration> get(final String id);
+
+    Mono<IntegrationPreviewList> listPreviews();
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/integration/service/IntegrationServiceImpl.java

@@ -0,0 +1,26 @@
+package org.opendatadiscovery.oddplatform.integration.service;
+
+import lombok.RequiredArgsConstructor;
+import org.opendatadiscovery.oddplatform.api.contract.model.Integration;
+import org.opendatadiscovery.oddplatform.api.contract.model.IntegrationPreviewList;
+import org.opendatadiscovery.oddplatform.integration.IntegrationRegistry;
+import org.opendatadiscovery.oddplatform.integration.mapper.IntegrationMapper;
+import org.springframework.stereotype.Service;
+import reactor.core.publisher.Mono;
+
+@Service
+@RequiredArgsConstructor
+public class IntegrationServiceImpl implements IntegrationService {
+    private final IntegrationRegistry integrationRegistry;
+    private final IntegrationMapper integrationMapper;
+
+    @Override
+    public Mono<Integration> get(final String id) {
+        return integrationRegistry.get(id).map(integrationMapper::map);
+    }
+
+    @Override
+    public Mono<IntegrationPreviewList> listPreviews() {
+        return integrationRegistry.list().collectList().map(integrationMapper::map);
+    }
+}