From c97703f4dd9e20c394b7fcbb4ce271b6e83a9d94 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 27 Aug 2024 17:10:47 +0530 Subject: [PATCH 01/19] feat: add dagger module --- cli/serve.go | 2 + modules/dagger/config.go | 90 ++++++++++++++++++++ modules/dagger/driver.go | 132 ++++++++++++++++++++++++++++++ modules/dagger/driver_output.go | 49 +++++++++++ modules/dagger/driver_plan.go | 69 ++++++++++++++++ modules/dagger/driver_sync.go | 12 +++ modules/dagger/module.go | 43 ++++++++++ modules/dagger/schema/config.json | 37 +++++++++ 8 files changed, 434 insertions(+) create mode 100644 modules/dagger/config.go create mode 100644 modules/dagger/driver.go create mode 100644 modules/dagger/driver_output.go create mode 100644 modules/dagger/driver_plan.go create mode 100644 modules/dagger/driver_sync.go create mode 100644 modules/dagger/module.go create mode 100644 modules/dagger/schema/config.json diff --git a/cli/serve.go b/cli/serve.go index 1224201..095f33b 100644 --- a/cli/serve.go +++ b/cli/serve.go @@ -14,6 +14,7 @@ import ( entropyserver "github.com/goto/entropy/internal/server" "github.com/goto/entropy/internal/store/postgres" "github.com/goto/entropy/modules" + "github.com/goto/entropy/modules/dagger" "github.com/goto/entropy/modules/firehose" "github.com/goto/entropy/modules/flink" "github.com/goto/entropy/modules/job" @@ -94,6 +95,7 @@ func setupRegistry() module.Registry { job.Module, kafka.Module, flink.Module, + dagger.Module, } registry := &modules.Registry{} diff --git a/modules/dagger/config.go b/modules/dagger/config.go new file mode 100644 index 0000000..584e04b --- /dev/null +++ b/modules/dagger/config.go @@ -0,0 +1,90 @@ +package dagger + +import ( + _ "embed" + "encoding/json" + "fmt" + + "github.com/goto/entropy/core/module" + "github.com/goto/entropy/modules" + "github.com/goto/entropy/pkg/errors" + "github.com/goto/entropy/pkg/validator" +) + +const helmReleaseNameMaxLength = 53 + +var ( + //go:embed schema/config.json + configSchemaRaw []byte + + validateConfig = validator.FromJSONSchema(configSchemaRaw) +) + +type UsageSpec struct { + CPU string `json:"cpu,omitempty" validate:"required"` + Memory string `json:"memory,omitempty" validate:"required"` +} + +type Resources struct { + TaskManager UsageSpec `json:"taskmanager,omitempty"` + JobManager UsageSpec `json:"jobmanager,omitempty"` +} + +type Stream struct { + SourceKafkaName string `json:"source_kafka_name,omitempty"` + ConsumerGroupId string `json:"consumer_group_id,omitempty"` +} + +type Config struct { + Resources Resources `json:"resources,omitempty"` + FlinkName string `json:"flink_name,omitempty"` + DeploymentID string `json:"deployment_id,omitempty"` + Streams []Stream `json:"streams,omitempty"` + JobId string `json:"job_id,omitempty"` + Savepoint any `json:"savepoint,omitempty"` + EnvVariables map[string]string `json:"env_variables,omitempty"` + ChartValues *ChartValues `json:"chart_values,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Namespace string `json:"namespace,omitempty"` + Replicas int `json:"replicas"` +} + +type ChartValues struct { + ChartVersion string `json:"chart_version" validate:"required"` +} + +func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverConf) (*Config, error) { + var cfg Config + err := json.Unmarshal(confJSON, &cfg) + if err != nil { + return nil, errors.ErrInvalid.WithMsgf("invalid config json").WithCausef(err.Error()) + } + + cfg.EnvVariables = modules.CloneAndMergeMaps(dc.EnvVariables, cfg.EnvVariables) + + if cfg.Replicas <= 0 { + cfg.Replicas = 1 + } + + if err := validateConfig(confJSON); err != nil { + return nil, err + } + + // note: enforce the kubernetes deployment name length limit. + if len(cfg.DeploymentID) == 0 { + cfg.DeploymentID = modules.SafeName(fmt.Sprintf("%s-%s", r.Project, r.Name), "-dagger", helmReleaseNameMaxLength) + } else if len(cfg.DeploymentID) > helmReleaseNameMaxLength { + return nil, errors.ErrInvalid.WithMsgf("deployment_id must not have more than 53 chars") + } + + cfg.Resources = dc.Resources + + if cfg.Namespace == "" { + //TODO: add error handling + kubeUrn := r.Spec.Dependencies[keyKubeDependency] + ns := dc.Namespace[kubeUrn] + cfg.Namespace = ns + } + + return &cfg, nil +} diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go new file mode 100644 index 0000000..7c7f9ee --- /dev/null +++ b/modules/dagger/driver.go @@ -0,0 +1,132 @@ +package dagger + +import ( + "context" + "encoding/json" + "time" + + "github.com/goto/entropy/core/module" + "github.com/goto/entropy/core/resource" + "github.com/goto/entropy/modules/kubernetes" + "github.com/goto/entropy/pkg/errors" + "github.com/goto/entropy/pkg/helm" + "github.com/goto/entropy/pkg/kube" +) + +const ( + stepReleaseCreate = "release_create" +) + +const ( + chartRepo = "https://goto.github.io/charts/" + chartName = "dagger" + imageRepo = "gotocompany/dagger" +) + +const ( + labelsConfKey = "labels" +) + +const defaultKey = "default" + +var defaultDriverConf = driverConf{ + Namespace: map[string]string{ + defaultKey: "dagger", + }, + ChartValues: ChartValues{ + ChartVersion: "0.1.0", + }, +} + +type daggerDriver struct { + timeNow func() time.Time + conf driverConf + kubeDeploy kubeDeployFn + kubeGetPod kubeGetPodFn +} + +type ( + kubeDeployFn func(ctx context.Context, isCreate bool, conf kube.Config, hc helm.ReleaseConfig) error + kubeGetPodFn func(ctx context.Context, conf kube.Config, ns string, labels map[string]string) ([]kube.Pod, error) +) + +type driverConf struct { + // Labels to be injected to the chart during deployment. Values can be Go templates. + Labels map[string]string `json:"labels,omitempty"` + + // Namespace is the kubernetes namespace where firehoses will be deployed. + Namespace map[string]string `json:"namespace" validate:"required"` + + // ChartValues is the chart and image version information. + ChartValues ChartValues `json:"chart_values" validate:"required"` + + EnvVariables map[string]string `json:"env_variables,omitempty"` + + Resources Resources `json:"resources" validate:"required"` + + JarURI string `json:"jar_uri" validate:"required"` + + // timeout value for a kube deployment run + KubeDeployTimeout int `json:"kube_deploy_timeout_seconds"` +} + +type Output struct { + State string `json:"state,omitempty"` + JMDeployStatus string `json:"jm_deploy_status,omitempty"` + JobStatus string `json:"job_status,omitempty"` + Namespace string `json:"namespace,omitempty"` + ReleaseName string `json:"release_name,omitempty"` + Pods []kube.Pod `json:"pods,omitempty"` +} + +type transientData struct { + PendingSteps []string `json:"pending_steps"` +} + +func (dd *daggerDriver) getHelmRelease(_ resource.Resource, conf Config, + _ kubernetes.Output, +) (*helm.ReleaseConfig, error) { + + rc := helm.DefaultReleaseConfig() + rc.Timeout = dd.conf.KubeDeployTimeout + rc.Name = conf.DeploymentID + rc.Repository = chartRepo + rc.Chart = chartName + rc.Namespace = conf.Namespace + rc.ForceUpdate = true + rc.Version = conf.ChartValues.ChartVersion + + rc.Values = map[string]any{ + labelsConfKey: dd.conf.Labels, + "replicaCount": conf.Replicas, + "dagger": map[string]any{ + "config": conf.EnvVariables, + "resources": conf.Resources, + }, + } + + return rc, nil +} + +func mergeChartValues(cur, newVal *ChartValues) (*ChartValues, error) { + if newVal == nil { + return cur, nil + } + + merged := ChartValues{ + ChartVersion: newVal.ChartVersion, + } + + return &merged, nil +} + +func readOutputData(exr module.ExpandedResource) (*Output, error) { + var curOut Output + if len(exr.Resource.State.Output) == 0 { + return &curOut, nil + } + if err := json.Unmarshal(exr.Resource.State.Output, &curOut); err != nil { + return nil, errors.ErrInternal.WithMsgf("corrupted output").WithCausef(err.Error()) + } + return &curOut, nil +} diff --git a/modules/dagger/driver_output.go b/modules/dagger/driver_output.go new file mode 100644 index 0000000..4f78b9d --- /dev/null +++ b/modules/dagger/driver_output.go @@ -0,0 +1,49 @@ +package dagger + +import ( + "context" + "encoding/json" + + "github.com/goto/entropy/core/module" + "github.com/goto/entropy/core/resource" + "github.com/goto/entropy/modules" + "github.com/goto/entropy/modules/kubernetes" + "github.com/goto/entropy/pkg/errors" +) + +func (dd *daggerDriver) Output(ctx context.Context, exr module.ExpandedResource) (json.RawMessage, error) { + output, err := readOutputData(exr) + if err != nil { + return nil, err + } + + conf, err := readConfig(exr, exr.Spec.Configs, dd.conf) + if err != nil { + return nil, errors.ErrInternal.WithCausef(err.Error()) + } + + var kubeOut kubernetes.Output + if err := json.Unmarshal(exr.Dependencies[keyKubeDependency].Output, &kubeOut); err != nil { + return nil, errors.ErrInternal.WithMsgf("invalid kube state").WithCausef(err.Error()) + } + + return dd.refreshOutput(ctx, exr.Resource, *conf, *output, kubeOut) +} + +func (dd *daggerDriver) refreshOutput(ctx context.Context, r resource.Resource, + conf Config, output Output, kubeOut kubernetes.Output, +) (json.RawMessage, error) { + rc, err := dd.getHelmRelease(r, conf, kubeOut) + if err != nil { + return nil, err + } + + pods, err := dd.kubeGetPod(ctx, kubeOut.Configs, rc.Namespace, map[string]string{"app": rc.Name}) + if err != nil { + return nil, errors.ErrInternal.WithCausef(err.Error()) + } + output.Pods = pods + output.Namespace = conf.Namespace + + return modules.MustJSON(output), nil +} diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go new file mode 100644 index 0000000..f4421b1 --- /dev/null +++ b/modules/dagger/driver_plan.go @@ -0,0 +1,69 @@ +package dagger + +import ( + "context" + "encoding/json" + + "github.com/goto/entropy/core/module" + "github.com/goto/entropy/core/resource" + "github.com/goto/entropy/modules" + "github.com/goto/entropy/modules/kubernetes" + "github.com/goto/entropy/pkg/errors" +) + +const SourceKafkaConsumerAutoOffsetReset = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET" + +func (dd *daggerDriver) Plan(_ context.Context, exr module.ExpandedResource, act module.ActionRequest) (*resource.Resource, error) { + switch act.Name { + case module.CreateAction: + return dd.planCreate(exr, act) + default: + return nil, nil + } +} + +func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.ActionRequest) (*resource.Resource, error) { + conf, err := readConfig(exr, act.Params, dd.conf) + if err != nil { + return nil, err + } + + chartVals, err := mergeChartValues(&dd.conf.ChartValues, conf.ChartValues) + if err != nil { + return nil, err + } + conf.ChartValues = chartVals + + immediately := dd.timeNow() + + exr.Resource.Spec.Configs = modules.MustJSON(conf) + + err = dd.validateHelmReleaseConfigs(exr, *conf) + if err != nil { + return nil, err + } + + exr.Resource.State = resource.State{ + Status: resource.StatusPending, + Output: modules.MustJSON(Output{ + Namespace: conf.Namespace, + ReleaseName: conf.DeploymentID, + }), + NextSyncAt: &immediately, + ModuleData: modules.MustJSON(transientData{ + PendingSteps: []string{stepReleaseCreate}, + }), + } + + return &exr.Resource, nil +} + +func (dd *daggerDriver) validateHelmReleaseConfigs(expandedResource module.ExpandedResource, config Config) error { + var kubeOut kubernetes.Output + if err := json.Unmarshal(expandedResource.Dependencies[keyKubeDependency].Output, &kubeOut); err != nil { + return errors.ErrInternal.WithMsgf("invalid kube state").WithCausef(err.Error()) + } + + _, err := dd.getHelmRelease(expandedResource.Resource, config, kubeOut) + return err +} diff --git a/modules/dagger/driver_sync.go b/modules/dagger/driver_sync.go new file mode 100644 index 0000000..e5c5958 --- /dev/null +++ b/modules/dagger/driver_sync.go @@ -0,0 +1,12 @@ +package dagger + +import ( + "context" + + "github.com/goto/entropy/core/module" + "github.com/goto/entropy/core/resource" +) + +func (dd *daggerDriver) Sync(ctx context.Context, exr module.ExpandedResource) (*resource.State, error) { + return nil, nil +} diff --git a/modules/dagger/module.go b/modules/dagger/module.go new file mode 100644 index 0000000..a3f73bf --- /dev/null +++ b/modules/dagger/module.go @@ -0,0 +1,43 @@ +package dagger + +import ( + _ "embed" + "encoding/json" + "time" + + "github.com/goto/entropy/core/module" + "github.com/goto/entropy/modules/kubernetes" + "github.com/goto/entropy/pkg/validator" +) + +const ( + keyKubeDependency = "kube_cluster" +) + +var Module = module.Descriptor{ + Kind: "dagger", + Dependencies: map[string]string{ + keyKubeDependency: kubernetes.Module.Kind, + }, + Actions: []module.ActionDesc{ + { + Name: module.CreateAction, + Description: "Creates a new dagger", + }, + }, + DriverFactory: func(confJSON json.RawMessage) (module.Driver, error) { + conf := defaultDriverConf // clone the default value + if err := json.Unmarshal(confJSON, &conf); err != nil { + return nil, err + } else if err := validator.TaggedStruct(conf); err != nil { + return nil, err + } + + return &daggerDriver{ + conf: conf, + timeNow: time.Now, + kubeDeploy: nil, + kubeGetPod: nil, + }, nil + }, +} diff --git a/modules/dagger/schema/config.json b/modules/dagger/schema/config.json new file mode 100644 index 0000000..453755b --- /dev/null +++ b/modules/dagger/schema/config.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": [ + "replicas", + "env_variables" + ], + "properties": { + "replicas": { + "type": "number", + "default": 1, + "minimum": 1 + }, + "deployment_id": { + "type": "string" + }, + "env_variables": { + "type": "object", + "additionalProperties": true, + "required": [ + "SINK_TYPE" + ], + "properties": { + "SINK_TYPE": { + "type": "string", + "enum": [ + "INFLUX", + "KAFKA", + "BIGQUERY" + ] + } + } + } + } + } + \ No newline at end of file From ab6fa4695a57e02cdbf4c62fd499fb3e01b1ef0c Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Mon, 9 Sep 2024 22:33:32 +0530 Subject: [PATCH 02/19] feat: add flink dep --- modules/dagger/config.go | 10 ++++++++-- modules/dagger/driver_output.go | 7 ++++--- modules/dagger/driver_plan.go | 8 ++++---- modules/dagger/module.go | 6 +++--- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index 584e04b..6af310f 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -7,6 +7,7 @@ import ( "github.com/goto/entropy/core/module" "github.com/goto/entropy/modules" + "github.com/goto/entropy/modules/flink" "github.com/goto/entropy/pkg/errors" "github.com/goto/entropy/pkg/validator" ) @@ -81,8 +82,13 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo if cfg.Namespace == "" { //TODO: add error handling - kubeUrn := r.Spec.Dependencies[keyKubeDependency] - ns := dc.Namespace[kubeUrn] + + var flinkOut flink.Output + if err := json.Unmarshal(r.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { + return nil, errors.ErrInternal.WithMsgf("invalid kube state").WithCausef(err.Error()) + } + + ns := flinkOut.KubeNamespace cfg.Namespace = ns } diff --git a/modules/dagger/driver_output.go b/modules/dagger/driver_output.go index 4f78b9d..7f2e819 100644 --- a/modules/dagger/driver_output.go +++ b/modules/dagger/driver_output.go @@ -7,6 +7,7 @@ import ( "github.com/goto/entropy/core/module" "github.com/goto/entropy/core/resource" "github.com/goto/entropy/modules" + "github.com/goto/entropy/modules/flink" "github.com/goto/entropy/modules/kubernetes" "github.com/goto/entropy/pkg/errors" ) @@ -22,12 +23,12 @@ func (dd *daggerDriver) Output(ctx context.Context, exr module.ExpandedResource) return nil, errors.ErrInternal.WithCausef(err.Error()) } - var kubeOut kubernetes.Output - if err := json.Unmarshal(exr.Dependencies[keyKubeDependency].Output, &kubeOut); err != nil { + var flinkOut flink.Output + if err := json.Unmarshal(exr.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { return nil, errors.ErrInternal.WithMsgf("invalid kube state").WithCausef(err.Error()) } - return dd.refreshOutput(ctx, exr.Resource, *conf, *output, kubeOut) + return dd.refreshOutput(ctx, exr.Resource, *conf, *output, flinkOut.KubeCluster) } func (dd *daggerDriver) refreshOutput(ctx context.Context, r resource.Resource, diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go index f4421b1..2253ef6 100644 --- a/modules/dagger/driver_plan.go +++ b/modules/dagger/driver_plan.go @@ -7,7 +7,7 @@ import ( "github.com/goto/entropy/core/module" "github.com/goto/entropy/core/resource" "github.com/goto/entropy/modules" - "github.com/goto/entropy/modules/kubernetes" + "github.com/goto/entropy/modules/flink" "github.com/goto/entropy/pkg/errors" ) @@ -59,11 +59,11 @@ func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.Actio } func (dd *daggerDriver) validateHelmReleaseConfigs(expandedResource module.ExpandedResource, config Config) error { - var kubeOut kubernetes.Output - if err := json.Unmarshal(expandedResource.Dependencies[keyKubeDependency].Output, &kubeOut); err != nil { + var flinkOut flink.Output + if err := json.Unmarshal(expandedResource.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { return errors.ErrInternal.WithMsgf("invalid kube state").WithCausef(err.Error()) } - _, err := dd.getHelmRelease(expandedResource.Resource, config, kubeOut) + _, err := dd.getHelmRelease(expandedResource.Resource, config, flinkOut.KubeCluster) return err } diff --git a/modules/dagger/module.go b/modules/dagger/module.go index a3f73bf..476adb5 100644 --- a/modules/dagger/module.go +++ b/modules/dagger/module.go @@ -6,18 +6,18 @@ import ( "time" "github.com/goto/entropy/core/module" - "github.com/goto/entropy/modules/kubernetes" + "github.com/goto/entropy/modules/flink" "github.com/goto/entropy/pkg/validator" ) const ( - keyKubeDependency = "kube_cluster" + keyFlinkDependency = "flink" ) var Module = module.Descriptor{ Kind: "dagger", Dependencies: map[string]string{ - keyKubeDependency: kubernetes.Module.Kind, + keyFlinkDependency: flink.Module.Kind, }, Actions: []module.ActionDesc{ { From 0fe1f260ac7f22369065903a061aab2b78776df9 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 10 Sep 2024 15:57:08 +0530 Subject: [PATCH 03/19] feat: add transformations --- modules/dagger/config.go | 173 +++++++++++++++++++++++++++++++++++---- 1 file changed, 155 insertions(+), 18 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index 6af310f..d106669 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -4,6 +4,8 @@ import ( _ "embed" "encoding/json" "fmt" + "strconv" + "strings" "github.com/goto/entropy/core/module" "github.com/goto/entropy/modules" @@ -13,6 +15,20 @@ import ( ) const helmReleaseNameMaxLength = 53 +const keyStreams = "STREAMS" +const keyFlinkJobID = "FLINK_JOB_ID" +const keySinkInfluxURL = "SINK_INFLUX_URL" +const keySinkInfluxPassword = "SINK_INFLUX_PASSWORD" +const keySinkInfluxDBName = "SINK_INFLUX_DB_NAME" +const keySinkInfluxUsername = "SINK_INFLUX_USERNAME" +const keySinkInfluxMeasurementName = "SINK_INFLUX_MEASUREMENT_NAME" +const keyRedisServer = "REDIS_SERVER" +const SourceKafkaConsumerConfigAutoCommitEnable = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE" +const SourceKafkaConsumerConfigAutoOffsetReset = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET" +const SinkTypeInflux = "influx" +const SinkTypeKafka = "kafka" +const keySinkKafkaBrokers = "SINK_KAFKA_BROKERS" +const keySinkKafkaStream = "SINK_KAFKA_STREAM" var ( //go:embed schema/config.json @@ -31,11 +47,6 @@ type Resources struct { JobManager UsageSpec `json:"jobmanager,omitempty"` } -type Stream struct { - SourceKafkaName string `json:"source_kafka_name,omitempty"` - ConsumerGroupId string `json:"consumer_group_id,omitempty"` -} - type Config struct { Resources Resources `json:"resources,omitempty"` FlinkName string `json:"flink_name,omitempty"` @@ -48,12 +59,32 @@ type Config struct { Deleted bool `json:"deleted,omitempty"` Namespace string `json:"namespace,omitempty"` Replicas int `json:"replicas"` + SinkType string `json:"sink_type"` } type ChartValues struct { ChartVersion string `json:"chart_version" validate:"required"` } +type SourceDetail struct { + SourceName string `json:"SOURCE_NAME"` + SourceType string `json:"SOURCE_TYPE"` +} + +type Stream struct { + InputSchemaTable string `json:"INPUT_SCHEMA_TABLE"` + SourceDetails []SourceDetail `json:"SOURCE_DETAILS"` + SourceKafkaConsumerConfigAutoCommitEnable string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE"` + SourceKafkaConsumerConfigAutoOffsetReset string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET"` + InputSchemaProtoClass string `json:"INPUT_SCHEMA_PROTO_CLASS"` + SourceKafkaTopicNames string `json:"SOURCE_KAFKA_TOPIC_NAMES"` + SourceKafkaName string `json:"SOURCE_KAFKA_NAME"` + InputSchemaEventTimestampFieldIndex string `json:"INPUT_SCHEMA_EVENT_TIMESTAMP_FIELD_INDEX"` + SourceParquetFileDateRange interface{} `json:"SOURCE_PARQUET_FILE_DATE_RANGE"` + SourceParquetFilePaths interface{} `json:"SOURCE_PARQUET_FILE_PATHS"` + SourceKafkaConsumerConfigGroupID string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_GROUP_ID"` +} + func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverConf) (*Config, error) { var cfg Config err := json.Unmarshal(confJSON, &cfg) @@ -61,16 +92,46 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo return nil, errors.ErrInvalid.WithMsgf("invalid config json").WithCausef(err.Error()) } + //transformation #1 + streamsJSON := cfg.EnvVariables[keyStreams] + var streams []Stream + err = json.Unmarshal([]byte(streamsJSON), &streams) + if err != nil { + return nil, errors.ErrInvalid.WithMsgf("invalid config json").WithCausef(err.Error()) + } + + for i := range streams { + if len(streams[i].SourceDetails) == 0 { + streams[i].SourceDetails = []SourceDetail{ + { + SourceName: "KAFKA_CONSUMER", + SourceType: "UNBOUNDED", + }, + } + } + } + + //transformation #2 cfg.EnvVariables = modules.CloneAndMergeMaps(dc.EnvVariables, cfg.EnvVariables) - if cfg.Replicas <= 0 { - cfg.Replicas = 1 + //transformation #3 + var flinkOut flink.Output + if err := json.Unmarshal(r.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { + return nil, errors.ErrInternal.WithMsgf("invalid kube state").WithCausef(err.Error()) } - if err := validateConfig(confJSON); err != nil { - return nil, err + if cfg.Namespace == "" { + ns := flinkOut.KubeNamespace + cfg.Namespace = ns } + //transformation #4 + //transform resource name to safe length + + //transformation #5 + cfg.EnvVariables[keyFlinkJobID] = r.Name + + //transformation #6 // note: enforce the kubernetes deployment name length limit. if len(cfg.DeploymentID) == 0 { cfg.DeploymentID = modules.SafeName(fmt.Sprintf("%s-%s", r.Project, r.Name), "-dagger", helmReleaseNameMaxLength) @@ -78,19 +139,95 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo return nil, errors.ErrInvalid.WithMsgf("deployment_id must not have more than 53 chars") } - cfg.Resources = dc.Resources + //transformation #7 + cfg.EnvVariables[keySinkInfluxURL] = flinkOut.Influx.URL + cfg.EnvVariables[keySinkInfluxPassword] = flinkOut.Influx.Password + cfg.EnvVariables[keySinkInfluxUsername] = flinkOut.Influx.Username + //TODO: add sink influx db name + //TODO: check if SINK_INFLUX_MEASUREMENT_NAME and REDIS_SERVER needs modification + + //transformation #8 + //Longbow related transformation skipped + + //transformation #9 and #11 + for i := range streams { + streams[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) + streams[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] + streams[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] + //TODO: add stream URL for key SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS + + cfg.Streams = append(cfg.Streams, Stream{ + SourceKafkaName: streams[i].SourceKafkaName, + SourceKafkaConsumerConfigGroupID: streams[i].SourceKafkaConsumerConfigGroupID, + }) + } - if cfg.Namespace == "" { - //TODO: add error handling + //transformation #10 + //this shall check if the project of the conf.EnvVars.STREAMS is same as that of the corresponding flink + //do we need to check this? - var flinkOut flink.Output - if err := json.Unmarshal(r.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { - return nil, errors.ErrInternal.WithMsgf("invalid kube state").WithCausef(err.Error()) - } + //transformation #12 + cfg.EnvVariables[keyStreams] = string(mustMarshalJSON(streams)) - ns := flinkOut.KubeNamespace - cfg.Namespace = ns + //transformation #13 + if cfg.SinkType == SinkTypeKafka { + cfg.EnvVariables[keySinkKafkaStream] = flinkOut.SinkKafkaStream + //TODO cfg.EnvVariables[keySinkKafkaBrokers] = stream URL + } + + //transformation #14 + cfg.Resources = mergeResources(dc.Resources, cfg.Resources) + + if cfg.Replicas <= 0 { + cfg.Replicas = 1 + } + + if err := validateConfig(confJSON); err != nil { + return nil, err } return &cfg, nil } +func incrementGroupId(groupId string, step int) string { + incrementNumberInString := func(number string) int { + num, _ := strconv.Atoi(number) + return num + step + } + + leftZeroPad := func(number int) string { + return fmt.Sprintf("%04d", number) + } + + getLastAndRestFromArray := func(arr []string) ([]string, string) { + return arr[:len(arr)-1], arr[len(arr)-1] + } + + parts := strings.Split(groupId, "-") + name, number := getLastAndRestFromArray(parts) + updatedNumber := leftZeroPad(incrementNumberInString(number)) + return strings.Join(append(name, updatedNumber), "-") +} + +func mustMarshalJSON(v interface{}) []byte { + data, err := json.Marshal(v) + if err != nil { + panic(fmt.Sprintf("failed to marshal JSON: %v", err)) + } + return data +} + +func mergeResources(defaultResources, currResources Resources) Resources { + if currResources.TaskManager.CPU == "" { + currResources.TaskManager.CPU = defaultResources.TaskManager.CPU + } + if currResources.TaskManager.Memory == "" { + currResources.TaskManager.Memory = defaultResources.TaskManager.Memory + } + if currResources.JobManager.CPU == "" { + currResources.JobManager.CPU = defaultResources.JobManager.CPU + } + if currResources.JobManager.Memory == "" { + currResources.JobManager.Memory = defaultResources.JobManager.Memory + } + return currResources +} From 8ca8432747e31eb6177fe46a953b512a840b1252 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 11 Sep 2024 12:50:00 +0530 Subject: [PATCH 04/19] fix: read stream from config root --- modules/dagger/config.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index d106669..67d2d0d 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -25,8 +25,8 @@ const keySinkInfluxMeasurementName = "SINK_INFLUX_MEASUREMENT_NAME" const keyRedisServer = "REDIS_SERVER" const SourceKafkaConsumerConfigAutoCommitEnable = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE" const SourceKafkaConsumerConfigAutoOffsetReset = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET" -const SinkTypeInflux = "influx" -const SinkTypeKafka = "kafka" +const SinkTypeInflux = "INFLUX" +const SinkTypeKafka = "KAFKA" const keySinkKafkaBrokers = "SINK_KAFKA_BROKERS" const keySinkKafkaStream = "SINK_KAFKA_STREAM" @@ -83,6 +83,7 @@ type Stream struct { SourceParquetFileDateRange interface{} `json:"SOURCE_PARQUET_FILE_DATE_RANGE"` SourceParquetFilePaths interface{} `json:"SOURCE_PARQUET_FILE_PATHS"` SourceKafkaConsumerConfigGroupID string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_GROUP_ID"` + SourceKafkaConsumerConfigBootstrapServers string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS"` } func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverConf) (*Config, error) { @@ -93,12 +94,7 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo } //transformation #1 - streamsJSON := cfg.EnvVariables[keyStreams] - var streams []Stream - err = json.Unmarshal([]byte(streamsJSON), &streams) - if err != nil { - return nil, errors.ErrInvalid.WithMsgf("invalid config json").WithCausef(err.Error()) - } + streams := cfg.Streams for i := range streams { if len(streams[i].SourceDetails) == 0 { @@ -150,6 +146,7 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo //Longbow related transformation skipped //transformation #9 and #11 + cfg.Streams = []Stream{} for i := range streams { streams[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) streams[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] From 010a36a60fb2926b8f5a1fc07426a022165c96b3 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 11 Sep 2024 15:32:37 +0530 Subject: [PATCH 05/19] feat: add Plan implementation --- modules/dagger/config.go | 39 +++++---- modules/dagger/driver.go | 140 +++++++++++++++++++++++++++------ modules/dagger/driver_plan.go | 2 + modules/dagger/driver_sync.go | 63 +++++++++++++++ modules/dagger/module.go | 30 ++++++- modules/flink/config.go | 1 + modules/flink/driver.go | 2 + modules/flink/driver_output.go | 1 + 8 files changed, 234 insertions(+), 44 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index 67d2d0d..48f4db4 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -48,22 +48,27 @@ type Resources struct { } type Config struct { - Resources Resources `json:"resources,omitempty"` - FlinkName string `json:"flink_name,omitempty"` - DeploymentID string `json:"deployment_id,omitempty"` - Streams []Stream `json:"streams,omitempty"` - JobId string `json:"job_id,omitempty"` - Savepoint any `json:"savepoint,omitempty"` - EnvVariables map[string]string `json:"env_variables,omitempty"` - ChartValues *ChartValues `json:"chart_values,omitempty"` - Deleted bool `json:"deleted,omitempty"` - Namespace string `json:"namespace,omitempty"` - Replicas int `json:"replicas"` - SinkType string `json:"sink_type"` + Resources Resources `json:"resources,omitempty"` + FlinkName string `json:"flink_name,omitempty"` + DeploymentID string `json:"deployment_id,omitempty"` + Streams []Stream `json:"streams,omitempty"` + JobId string `json:"job_id,omitempty"` + Savepoint any `json:"savepoint,omitempty"` + EnvVariables map[string]string `json:"env_variables,omitempty"` + ChartValues *ChartValues `json:"chart_values,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Namespace string `json:"namespace,omitempty"` + Replicas int `json:"replicas"` + SinkType string `json:"sink_type"` + PrometheusURL string `json:"prometheus_url"` + JarURI string `json:"jar_uri"` } type ChartValues struct { - ChartVersion string `json:"chart_version" validate:"required"` + ImageRepository string `json:"image_repository" validate:"required"` + ImageTag string `json:"image_tag" validate:"required"` + ChartVersion string `json:"chart_version" validate:"required"` + ImagePullPolicy string `json:"image_pull_policy"` } type SourceDetail struct { @@ -113,7 +118,7 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo //transformation #3 var flinkOut flink.Output if err := json.Unmarshal(r.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { - return nil, errors.ErrInternal.WithMsgf("invalid kube state").WithCausef(err.Error()) + return nil, errors.ErrInternal.WithMsgf("invalid flink state").WithCausef(err.Error()) } if cfg.Namespace == "" { @@ -148,7 +153,9 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo //transformation #9 and #11 cfg.Streams = []Stream{} for i := range streams { - streams[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) + if streams[i].SourceKafkaConsumerConfigGroupID == "" { + streams[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) + } streams[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] streams[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] //TODO: add stream URL for key SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS @@ -175,6 +182,8 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo //transformation #14 cfg.Resources = mergeResources(dc.Resources, cfg.Resources) + cfg.PrometheusURL = flinkOut.PrometheusURL + if cfg.Replicas <= 0 { cfg.Replicas = 1 } diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go index 7c7f9ee..f78423c 100644 --- a/modules/dagger/driver.go +++ b/modules/dagger/driver.go @@ -1,12 +1,16 @@ package dagger import ( + "bytes" "context" + "encoding/base64" "encoding/json" + "html/template" "time" "github.com/goto/entropy/core/module" "github.com/goto/entropy/core/resource" + "github.com/goto/entropy/modules" "github.com/goto/entropy/modules/kubernetes" "github.com/goto/entropy/pkg/errors" "github.com/goto/entropy/pkg/helm" @@ -24,7 +28,15 @@ const ( ) const ( - labelsConfKey = "labels" + labelsConfKey = "extra_labels" + + labelDeployment = "deployment" + labelOrchestrator = "orchestrator" + labelURN = "urn" + labelName = "name" + labelNamespace = "namespace" + + orchestratorLabelValue = "entropy" ) const defaultKey = "default" @@ -83,31 +95,6 @@ type transientData struct { PendingSteps []string `json:"pending_steps"` } -func (dd *daggerDriver) getHelmRelease(_ resource.Resource, conf Config, - _ kubernetes.Output, -) (*helm.ReleaseConfig, error) { - - rc := helm.DefaultReleaseConfig() - rc.Timeout = dd.conf.KubeDeployTimeout - rc.Name = conf.DeploymentID - rc.Repository = chartRepo - rc.Chart = chartName - rc.Namespace = conf.Namespace - rc.ForceUpdate = true - rc.Version = conf.ChartValues.ChartVersion - - rc.Values = map[string]any{ - labelsConfKey: dd.conf.Labels, - "replicaCount": conf.Replicas, - "dagger": map[string]any{ - "config": conf.EnvVariables, - "resources": conf.Resources, - }, - } - - return rc, nil -} - func mergeChartValues(cur, newVal *ChartValues) (*ChartValues, error) { if newVal == nil { return cur, nil @@ -130,3 +117,104 @@ func readOutputData(exr module.ExpandedResource) (*Output, error) { } return &curOut, nil } + +func readTransientData(exr module.ExpandedResource) (*transientData, error) { + if len(exr.Resource.State.ModuleData) == 0 { + return &transientData{}, nil + } + + var modData transientData + if err := json.Unmarshal(exr.Resource.State.ModuleData, &modData); err != nil { + return nil, errors.ErrInternal.WithMsgf("corrupted transient data").WithCausef(err.Error()) + } + return &modData, nil +} + +func (dd *daggerDriver) getHelmRelease(res resource.Resource, conf Config, + kubeOut kubernetes.Output, +) (*helm.ReleaseConfig, error) { + + entropyLabels := map[string]string{ + labelDeployment: conf.DeploymentID, + labelOrchestrator: orchestratorLabelValue, + } + + otherLabels := map[string]string{ + labelURN: res.URN, + labelName: res.Name, + labelNamespace: conf.Namespace, + } + + deploymentLabels, err := renderTpl(dd.conf.Labels, modules.CloneAndMergeMaps(res.Labels, modules.CloneAndMergeMaps(entropyLabels, otherLabels))) + if err != nil { + return nil, err + } + + rc := helm.DefaultReleaseConfig() + rc.Timeout = dd.conf.KubeDeployTimeout + rc.Name = conf.DeploymentID + rc.Repository = chartRepo + rc.Chart = chartName + rc.Namespace = conf.Namespace + rc.ForceUpdate = true + rc.Version = conf.ChartValues.ChartVersion + + imageRepository := dd.conf.ChartValues.ImageRepository + if conf.ChartValues.ImageRepository != "" { + imageRepository = conf.ChartValues.ImageRepository + } + + envVarsJSON, err := json.Marshal(conf.EnvVariables) + if err != nil { + return nil, errors.ErrInternal.WithMsgf("failed to marshal env variables").WithCausef(err.Error()) + } + + encodedEnvVars := base64.StdEncoding.EncodeToString(envVarsJSON) + programArgs := encodedEnvVars + + rc.Values = map[string]any{ + labelsConfKey: modules.CloneAndMergeMaps(deploymentLabels, entropyLabels), + "image": imageRepository, + "deployment_id": conf.DeploymentID, + "configurations": map[string]any{ + "FLINK_PARALLELISM": conf.Replicas, + }, + "projectID": res.Project, + "name": res.Name, + "team": res.Labels["team"], //TODO: improve handling this case + "flink_name": conf.FlinkName, + "prometheus_url": conf.PrometheusURL, + "resources": conf.Resources, + "jar_uri": conf.JarURI, + "programArgs": "--encodedArgs" + programArgs, + } + + return rc, nil +} + +// TODO: move this to pkg +func renderTpl(labelsTpl map[string]string, labelsValues map[string]string) (map[string]string, error) { + const useZeroValueForMissingKey = "missingkey=zero" + + finalLabels := map[string]string{} + for k, v := range labelsTpl { + var buf bytes.Buffer + t, err := template.New("").Option(useZeroValueForMissingKey).Parse(v) + if err != nil { + return nil, errors.ErrInvalid. + WithMsgf("label template for '%s' is invalid", k).WithCausef(err.Error()) + } else if err := t.Execute(&buf, labelsValues); err != nil { + return nil, errors.ErrInvalid. + WithMsgf("failed to render label template").WithCausef(err.Error()) + } + + // allow empty values + // labelVal := strings.TrimSpace(buf.String()) + // if labelVal == "" { + // continue + // } + + finalLabels[k] = buf.String() + } + return finalLabels, nil +} diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go index 2253ef6..27e6ecc 100644 --- a/modules/dagger/driver_plan.go +++ b/modules/dagger/driver_plan.go @@ -43,6 +43,8 @@ func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.Actio return nil, err } + conf.JarURI = dd.conf.JarURI + exr.Resource.State = resource.State{ Status: resource.StatusPending, Output: modules.MustJSON(Output{ diff --git a/modules/dagger/driver_sync.go b/modules/dagger/driver_sync.go index e5c5958..e2ba30d 100644 --- a/modules/dagger/driver_sync.go +++ b/modules/dagger/driver_sync.go @@ -2,11 +2,74 @@ package dagger import ( "context" + "encoding/json" "github.com/goto/entropy/core/module" "github.com/goto/entropy/core/resource" + "github.com/goto/entropy/modules" + "github.com/goto/entropy/modules/flink" + "github.com/goto/entropy/modules/kubernetes" + "github.com/goto/entropy/pkg/errors" ) func (dd *daggerDriver) Sync(ctx context.Context, exr module.ExpandedResource) (*resource.State, error) { + modData, err := readTransientData(exr) + if err != nil { + return nil, err + } + + conf, err := readConfig(exr, exr.Spec.Configs, dd.conf) + if err != nil { + return nil, errors.ErrInternal.WithCausef(err.Error()) + } + + var flinkOut flink.Output + if err := json.Unmarshal(exr.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { + return nil, errors.ErrInternal.WithMsgf("invalid flink state").WithCausef(err.Error()) + } + + finalState := resource.State{ + Status: resource.StatusPending, + Output: exr.Resource.State.Output, + } + + if len(modData.PendingSteps) > 0 { + pendingStep := modData.PendingSteps[0] + modData.PendingSteps = modData.PendingSteps[1:] + + switch pendingStep { + case stepReleaseCreate: + isCreate := pendingStep == stepReleaseCreate + if err := dd.releaseSync(ctx, exr.Resource, isCreate, *conf, flinkOut.KubeCluster); err != nil { + return nil, err + } + default: + return nil, errors.ErrInternal.WithMsgf("unknown step: '%s'", pendingStep) + } + + // we have more pending states, so enqueue resource for another sync + // as soon as possible. + immediately := dd.timeNow() + finalState.NextSyncAt = &immediately + finalState.ModuleData = modules.MustJSON(modData) + + return &finalState, nil + } + return nil, nil + +} + +func (dd *daggerDriver) releaseSync(ctx context.Context, r resource.Resource, + isCreate bool, conf Config, kubeOut kubernetes.Output, +) error { + rc, err := dd.getHelmRelease(r, conf, kubeOut) + if err != nil { + return err + } + + if err := dd.kubeDeploy(ctx, isCreate, kubeOut.Configs, *rc); err != nil { + return errors.ErrInternal.WithCausef(err.Error()) + } + return nil } diff --git a/modules/dagger/module.go b/modules/dagger/module.go index 476adb5..7323ab6 100644 --- a/modules/dagger/module.go +++ b/modules/dagger/module.go @@ -1,13 +1,17 @@ package dagger import ( + "context" _ "embed" "encoding/json" "time" "github.com/goto/entropy/core/module" "github.com/goto/entropy/modules/flink" + "github.com/goto/entropy/pkg/helm" + "github.com/goto/entropy/pkg/kube" "github.com/goto/entropy/pkg/validator" + "helm.sh/helm/v3/pkg/release" ) const ( @@ -34,9 +38,29 @@ var Module = module.Descriptor{ } return &daggerDriver{ - conf: conf, - timeNow: time.Now, - kubeDeploy: nil, + conf: conf, + timeNow: time.Now, + kubeDeploy: func(_ context.Context, isCreate bool, kubeConf kube.Config, hc helm.ReleaseConfig) error { + canUpdate := func(rel *release.Release) bool { + curLabels, ok := rel.Config[labelsConfKey].(map[string]any) + if !ok { + return false + } + newLabels, ok := hc.Values[labelsConfKey].(map[string]string) + if !ok { + return false + } + + isManagedByEntropy := curLabels[labelOrchestrator] == orchestratorLabelValue + isSameDeployment := curLabels[labelDeployment] == newLabels[labelDeployment] + + return isManagedByEntropy && isSameDeployment + } + + helmCl := helm.NewClient(&helm.Config{Kubernetes: kubeConf}) + _, errHelm := helmCl.Upsert(&hc, canUpdate) + return errHelm + }, kubeGetPod: nil, }, nil }, diff --git a/modules/flink/config.go b/modules/flink/config.go index 24a06a2..50401f1 100644 --- a/modules/flink/config.go +++ b/modules/flink/config.go @@ -26,6 +26,7 @@ type Config struct { KubeNamespace string `json:"kube_namespace,omitempty"` Influx Influx `json:"influx,omitempty"` SinkKafkaStream string `json:"sink_kafka_stream,omitempty"` + PrometheusURL string `json:"prometheus_url,omitempty"` } func readConfig(_ resource.Resource, confJSON json.RawMessage, dc driverConf) (*Config, error) { diff --git a/modules/flink/driver.go b/modules/flink/driver.go index 64a5365..192df3d 100644 --- a/modules/flink/driver.go +++ b/modules/flink/driver.go @@ -16,6 +16,7 @@ type driverConf struct { Influx Influx `json:"influx,omitempty"` SinkKafkaStream string `json:"sink_kafka_stream,omitempty"` KubeNamespace string `json:"kube_namespace,omitempty"` + PrometheusURL string `json:"prometheus_url,omitempty"` } type Output struct { @@ -23,6 +24,7 @@ type Output struct { KubeNamespace string `json:"kube_namespace,omitempty"` Influx Influx `json:"influx,omitempty"` SinkKafkaStream string `json:"sink_kafka_stream,omitempty"` + PrometheusURL string `json:"prometheus_url,omitempty"` } func readOutputData(exr module.ExpandedResource) (*Output, error) { diff --git a/modules/flink/driver_output.go b/modules/flink/driver_output.go index 2182308..01a58b0 100644 --- a/modules/flink/driver_output.go +++ b/modules/flink/driver_output.go @@ -33,6 +33,7 @@ func (fd *flinkDriver) Output(ctx context.Context, exr module.ExpandedResource) output.Influx = conf.Influx output.KubeNamespace = conf.KubeNamespace output.SinkKafkaStream = conf.SinkKafkaStream + output.PrometheusURL = conf.PrometheusURL return modules.MustJSON(output), nil } From 7a048eddef892360c81d389130ff2876642aaf41 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Fri, 13 Sep 2024 15:02:33 +0530 Subject: [PATCH 06/19] fix: chart values --- modules/dagger/driver.go | 33 ++++++++++++++++++++++----------- modules/dagger/driver_plan.go | 3 +-- modules/dagger/driver_sync.go | 2 +- pkg/helm/helm.go | 11 ++++++++--- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go index f78423c..70c5e15 100644 --- a/modules/dagger/driver.go +++ b/modules/dagger/driver.go @@ -3,7 +3,6 @@ package dagger import ( "bytes" "context" - "encoding/base64" "encoding/json" "html/template" "time" @@ -23,7 +22,7 @@ const ( const ( chartRepo = "https://goto.github.io/charts/" - chartName = "dagger" + chartName = "dagger-deployment-chart" imageRepo = "gotocompany/dagger" ) @@ -164,29 +163,41 @@ func (dd *daggerDriver) getHelmRelease(res resource.Resource, conf Config, imageRepository = conf.ChartValues.ImageRepository } - envVarsJSON, err := json.Marshal(conf.EnvVariables) + _, err = json.Marshal(conf.EnvVariables) if err != nil { return nil, errors.ErrInternal.WithMsgf("failed to marshal env variables").WithCausef(err.Error()) } - encodedEnvVars := base64.StdEncoding.EncodeToString(envVarsJSON) - programArgs := encodedEnvVars + //encodedEnvVars := base64.StdEncoding.EncodeToString(envVarsJSON) + //programArgs := encodedEnvVars rc.Values = map[string]any{ labelsConfKey: modules.CloneAndMergeMaps(deploymentLabels, entropyLabels), "image": imageRepository, "deployment_id": conf.DeploymentID, - "configurations": map[string]any{ + "configuration": map[string]any{ "FLINK_PARALLELISM": conf.Replicas, }, "projectID": res.Project, "name": res.Name, - "team": res.Labels["team"], //TODO: improve handling this case - "flink_name": conf.FlinkName, + "team": res.Labels["team"], //TODO: improve handling this case + "flink_name": "g-pilotdata-gl-playground-operator-v2", //TODO: use this from flink resource "prometheus_url": conf.PrometheusURL, - "resources": conf.Resources, - "jar_uri": conf.JarURI, - "programArgs": "--encodedArgs" + programArgs, + "resources": map[string]any{ + "jobmanager": map[string]any{ + "cpu": conf.Resources.JobManager.CPU, + "memory": conf.Resources.JobManager.Memory, + }, + "taskmanager": map[string]any{ + "cpu": conf.Resources.TaskManager.CPU, + "memory": conf.Resources.JobManager.Memory, + }, + }, + "jarURI": conf.JarURI, + //TODO: build programArgs from EnvVariables + "programArgs": []string{"--encodedArgs", "WyItLVNPVVJDRV9LQUZLQV9DT05TVU1FX0xBUkdFX01FU1NBR0VfRU5BQkxFIixmYWxzZSwiLS1FTkFCTEVfU1RFTkNJTF9VUkwiLHRydWUsIi0tRkxJTktfSk9CX0lEIiwiZy1waWxvdGRhdGEtZ2wtdGVzdC1kb2NrZXItZXJyb3ItZGFnZ2VyIiwiLS1TSU5LX0lORkxVWF9CQVRDSF9TSVpFIiwxMDAsIi0tU0lOS19JTkZMVVhfREJfTkFNRSIsIkRBR0dFUlNfUExBWUdST1VORCIsIi0tU0lOS19JTkZMVVhfRkxVU0hfRFVSQVRJT05fTVMiLDEwMDAsIi0tU0lOS19JTkZMVVhfTUVBU1VSRU1FTlRfTkFNRSIsInRlc3QtZG9ja2VyLWVycm9yIiwiLS1TSU5LX0lORkxVWF9QQVNTV09SRCIsInJvb3QiLCItLVNJTktfSU5GTFVYX1JFVEVOVElPTl9QT0xJQ1kiLCJhdXRvZ2VuIiwiLS1TSU5LX0lORkxVWF9VUkwiLCJodHRwOi8vMTAuODQuNTIuODk6ODA4NiIsIi0tU0lOS19JTkZMVVhfVVNFUk5BTUUiLCJyb290IiwiLS1QUk9DRVNTT1JfTE9OR0JPV19HQ1BfSU5TVEFOQ0VfSUQiLCJ0ZXN0IiwiLS1QUk9DRVNTT1JfTE9OR0JPV19HQ1BfUFJPSkVDVF9JRCIsInRlc3QiLCItLVNJTktfS0FGS0FfQlJPS0VSUyIsIiIsIi0tU0lOS19LQUZLQV9UT1BJQyIsIiIsIi0tT1VUUFVUX1BST1RPX0NMQVNTX1BSRUZJWCIsIiIsIi0tU0lOS19LQUZLQV9QUk9UT19LRVkiLCIiLCItLVNJTktfS0FGS0FfUFJPVE9fTUVTU0FHRSIsIiIsIi0tU0lOS19LQUZLQV9TVFJFQU0iLCIiLCItLUZMSU5LX1BBUkFMTEVMSVNNIiwxLCItLVBPUlRBTF9WRVJTSU9OIiwiMiIsIi0tUFJPQ0VTU09SX1BSRVBST0NFU1NPUl9DT05GSUciLCIiLCItLVBST0NFU1NPUl9QUkVQUk9DRVNTT1JfRU5BQkxFIixmYWxzZSwiLS1QUk9DRVNTT1JfUE9TVFBST0NFU1NPUl9DT05GSUciLCIiLCItLVBST0NFU1NPUl9QT1NUUFJPQ0VTU09SX0VOQUJMRSIsZmFsc2UsIi0tU0lOS19LQUZLQV9QUk9EVUNFX0xBUkdFX01FU1NBR0VfRU5BQkxFIixmYWxzZSwiLS1SRURJU19TRVJWRVIiLCIiLCItLUZMSU5LX1JPV1RJTUVfQVRUUklCVVRFX05BTUUiLCJyb3d0aW1lIiwiLS1TSU5LX1RZUEUiLCJpbmZsdXgiLCItLUZMSU5LX1NRTF9RVUVSWSIsIlNFTEVDVCAqIEZST00gZGF0YV9zdHJlYW1zXzAiLCItLVNDSEVNQV9SRUdJU1RSWV9TVEVOQ0lMX0VOQUJMRSIsdHJ1ZSwiLS1TQ0hFTUFfUkVHSVNUUllfU1RFTkNJTF9VUkxTIiwiaHR0cDovL2ctZ29kYXRhLXN5c3RlbXMtc3RlbmNpbC12MWJldGExLWluZ3Jlc3MuZ29sYWJzLmlvL3YxYmV0YTEvbmFtZXNwYWNlcy9nb2play9zY2hlbWFzL2VzYi1sb2ctZW50aXRpZXMvdmVyc2lvbnMvOCIsIi0tU1RSRUFNUyIsIlt7XCJJTlBVVF9TQ0hFTUFfRVZFTlRfVElNRVNUQU1QX0ZJRUxEX0lOREVYXCI6XCI1XCIsXCJJTlBVVF9TQ0hFTUFfUFJPVE9fQ0xBU1NcIjpcImdvamVrLmVzYi5ib29raW5nLkJvb2tpbmdMb2dNZXNzYWdlXCIsXCJJTlBVVF9TQ0hFTUFfVEFCTEVcIjpcImRhdGFfc3RyZWFtc18wXCIsXCJTT1VSQ0VfREVUQUlMU1wiOlt7XCJTT1VSQ0VfTkFNRVwiOlwiS0FGS0FfQ09OU1VNRVJcIixcIlNPVVJDRV9UWVBFXCI6XCJVTkJPVU5ERURcIn1dLFwiU09VUkNFX0tBRktBX0NPTlNVTUVSX0NPTkZJR19BVVRPX0NPTU1JVF9FTkFCTEVcIjpcImZhbHNlXCIsXCJTT1VSQ0VfS0FGS0FfQ09OU1VNRVJfQ09ORklHX0FVVE9fT0ZGU0VUX1JFU0VUXCI6XCJsYXRlc3RcIixcIlNPVVJDRV9LQUZLQV9DT05TVU1FUl9DT05GSUdfQk9PVFNUUkFQX1NFUlZFUlNcIjpcImctcGlsb3RkYXRhLWdsLW1haW5zdHJlYW0ta2Fma2EuZ29sYWJzLmlvOjY2NjhcIixcIlNPVVJDRV9LQUZLQV9DT05TVU1FUl9DT05GSUdfR1JPVVBfSURcIjpcImctcGlsb3RkYXRhLWdsLWRhZ2dlci10ZXN0LTAxLWRhZ2dlci0wMDAxXCIsXCJTT1VSQ0VfS0FGS0FfTkFNRVwiOlwiZy1waWxvdGRhdGEtZ2wtbWFpbnN0cmVhbVwiLFwiU09VUkNFX0tBRktBX1RPUElDX05BTUVTXCI6XCJhZ2dyZWdhdGVkZGVtYW5kLXByaWNpbmd0ZXN0XCIsXCJTT1VSQ0VfUEFSUVVFVF9GSUxFX0RBVEVfUkFOR0VcIjpudWxsLFwiU09VUkNFX1BBUlFVRVRfRklMRV9QQVRIU1wiOm51bGx9XSIsIi0tRkxJTktfV0FURVJNQVJLX0RFTEFZX01TIiwxMDAwLCItLUZMSU5LX1dBVEVSTUFSS19JTlRFUlZBTF9NUyIsNjAwMDAsIi0tVURGX0RBUlRfR0NTX0JVQ0tFVF9JRCIsInAtZ29kYXRhLWRhZ2dlcnMtZGFydHMtc3RvcmFnZSIsIi0tVURGX0RBUlRfR0NTX1BST0pFQ1RfSUQiLCJnb2RhdGEtcHJvZHVjdGlvbiIsIi0tRlVOQ1RJT05fRkFDVE9SWV9DTEFTU0VTIiwiY29tLmdvdG9jb21wYW55LmRhZ2dlci5mdW5jdGlvbnMudWRmcy5mYWN0b3JpZXMuRnVuY3Rpb25GYWN0b3J5LGNvbS5nb2play5kZS5mbHVkLnVkZnMuZmFjdG9yaWVzLkZ1bmN0aW9uRmFjdG9yeSIsIi0tUFlUSE9OX1VERl9FTkFCTEUiLGZhbHNlLCItLVBZVEhPTl9VREZfQ09ORklHIiwie1wiUFlUSE9OX0ZJTEVTXCI6IFwiZ3M6Ly9nb2RhdGEtZGFnZ2VyL3B5dGhvbi9tYXN0ZXIvbGF0ZXN0L3B5dGhvbl91ZGZzLnppcFwiLFwiUFlUSE9OX1JFUVVJUkVNRU5UU1wiOiBcImdzOi8vZ29kYXRhLWRhZ2dlci9weXRob24vbWFzdGVyL2xhdGVzdC9yZXF1aXJlbWVudHMudHh0XCIsXCJQWVRIT05fQVJDSElWRVNcIjogXCJnczovL2dvZGF0YS1kYWdnZXIvcHl0aG9uL21hc3Rlci9sYXRlc3QvZGF0YS56aXAjZGF0YVwiLFwiUFlUSE9OX0ZOX0VYRUNVVElPTl9BUlJPV19CQVRDSF9TSVpFXCI6IFwiMTAwMDBcIixcIlBZVEhPTl9GTl9FWEVDVVRJT05fQlVORExFX1NJWkVcIjogXCIxMDAwMDBcIixcIlBZVEhPTl9GTl9FWEVDVVRJT05fQlVORExFX1RJTUVcIjogXCIxMDAwXCJ9IiwiLS1TSU5LX0JJR1FVRVJZX0dPT0dMRV9DTE9VRF9QUk9KRUNUX0lEIiwiIiwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX05BTUUiLCIiLCItLVNJTktfQklHUVVFUllfREFUQVNFVF9MQUJFTFMiLCIiLCItLVNJTktfQklHUVVFUllfVEFCTEVfTEFCRUxTIiwiIiwiLS1TSU5LX0JJR1FVRVJZX0RBVEFTRVRfTkFNRSIsIiIsIi0tU0lOS19CSUdRVUVSWV9DUkVERU5USUFMX1BBVEgiLCIvdmFyL3NlY3JldHMvZ29vZ2xlL2djcF9rZXkuanNvbiIsIi0tU0lOS19CSUdRVUVSWV9UQUJMRV9QQVJUSVRJT05JTkdfRU5BQkxFIixmYWxzZSwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX1BBUlRJVElPTl9LRVkiLCIiLCItLVNJTktfQklHUVVFUllfUk9XX0lOU0VSVF9JRF9FTkFCTEUiLGZhbHNlLCItLVNJTktfQklHUVVFUllfQ0xJRU5UX1JFQURfVElNRU9VVF9NUyIsLTEsIi0tU0lOS19CSUdRVUVSWV9DTElFTlRfQ09OTkVDVF9USU1FT1VUX01TIiwtMSwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX1BBUlRJVElPTl9FWFBJUllfTVMiLC0xLCItLVNJTktfQklHUVVFUllfREFUQVNFVF9MT0NBVElPTiIsImFzaWEtc291dGhlYXN0MSIsIi0tU0lOS19CSUdRVUVSWV9NRVRBREFUQV9OQU1FU1BBQ0UiLCIiLCItLVNJTktfQklHUVVFUllfQUREX01FVEFEQVRBX0VOQUJMRUQiLGZhbHNlLCItLVNJTktfQklHUVVFUllfTUVUQURBVEFfQ09MVU1OU19UWVBFUyIsIiIsIi0tU0lOS19NRVRSSUNTX0FQUExJQ0FUSU9OX1BSRUZJWCIsImRhZ2dlcl8iLCItLVNJTktfQklHUVVFUllfQkFUQ0hfU0laRSIsIiIsIi0tU0lOS19DT05ORUNUT1JfU0NIRU1BX1BST1RPX01FU1NBR0VfQ0xBU1MiLCIiLCItLVNJTktfQ09OTkVDVE9SX1NDSEVNQV9QUk9UT19LRVlfQ0xBU1MiLCIiLCItLVNJTktfQ09OTkVDVE9SX1NDSEVNQV9EQVRBX1RZUEUiLCJQUk9UT0JVRiIsIi0tU0lOS19DT05ORUNUT1JfU0NIRU1BX01FU1NBR0VfTU9ERSIsIkxPR19NRVNTQUdFIiwiLS1TSU5LX0NPTk5FQ1RPUl9TQ0hFTUFfUFJPVE9fQUxMT1dfVU5LTk9XTl9GSUVMRFNfRU5BQkxFIixmYWxzZSwiLS1TSU5LX0VSUk9SX1RZUEVTX0ZPUl9GQUlMVVJFIiwiIiwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX0NMVVNURVJJTkdfRU5BQkxFIixmYWxzZSwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX0NMVVNURVJJTkdfS0VZUyIsIiIsIi0tU0NIRU1BX1JFR0lTVFJZX1NURU5DSUxfQ0FDSEVfQVVUT19SRUZSRVNIIixmYWxzZSwiLS1TQ0hFTUFfUkVHSVNUUllfU1RFTkNJTF9SRUZSRVNIX1NUUkFURUdZIiwiTE9OR19QT0xMSU5HIiwiLS1TQ0hFTUFfUkVHSVNUUllfU1RFTkNJTF9DQUNIRV9UVExfTVMiLCI5MDAwMDAiLCItLVNJTktfS0FGS0FfTElOR0VSX01TIiwiMCIsIi0tU0NIRU1BX1JFR0lTVFJZX1NURU5DSUxfRkVUQ0hfUkVUUklFUyIsIjQiLCItLVNDSEVNQV9SRUdJU1RSWV9TVEVOQ0lMX0ZFVENIX0JBQ0tPRkZfTUlOX01TIiwiNjAwMDAiLCItLVNDSEVNQV9SRUdJU1RSWV9TVEVOQ0lMX0ZFVENIX1RJTUVPVVRfTVMiLCIxMDAwIl0="}, + "state": "running", + "namespace": conf.Namespace, } return rc, nil diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go index 27e6ecc..91c12f8 100644 --- a/modules/dagger/driver_plan.go +++ b/modules/dagger/driver_plan.go @@ -35,6 +35,7 @@ func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.Actio conf.ChartValues = chartVals immediately := dd.timeNow() + conf.JarURI = dd.conf.JarURI exr.Resource.Spec.Configs = modules.MustJSON(conf) @@ -43,8 +44,6 @@ func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.Actio return nil, err } - conf.JarURI = dd.conf.JarURI - exr.Resource.State = resource.State{ Status: resource.StatusPending, Output: modules.MustJSON(Output{ diff --git a/modules/dagger/driver_sync.go b/modules/dagger/driver_sync.go index e2ba30d..9398a14 100644 --- a/modules/dagger/driver_sync.go +++ b/modules/dagger/driver_sync.go @@ -56,7 +56,7 @@ func (dd *daggerDriver) Sync(ctx context.Context, exr module.ExpandedResource) ( return &finalState, nil } - return nil, nil + return &finalState, nil } diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index c032a1f..2744129 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -70,26 +70,31 @@ func (p *Client) doCreate(actionConfig *action.Configuration, config *ReleaseCon } act := action.NewInstall(actionConfig) - act.Wait = config.Wait - act.DryRun = false + act.Wait = true + act.IncludeCRDs = true + act.SkipCRDs = false act.Timeout = time.Duration(config.Timeout) * time.Second act.Replace = config.Replace act.OutputDir = "" act.Namespace = config.Namespace act.ClientOnly = false act.Description = config.Description - act.WaitForJobs = config.WaitForJobs + act.WaitForJobs = true act.ReleaseName = config.Name act.GenerateName = false act.NameTemplate = "" act.CreateNamespace = config.CreateNamespace act.ChartPathOptions = *chartPathOpts + act.DryRun = false rel, err := act.Run(fetchedChart, config.Values) if err != nil { return nil, errors.ErrInternal.WithMsgf("create-release failed").WithCausef(err.Error()) } + //TODO: remove this + fmt.Println("Release manifest\n", rel) + return &Result{ Config: config, Release: rel, From a2a60678cd099996d501af218e07bbedbab743b1 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 17 Sep 2024 18:00:33 +0530 Subject: [PATCH 07/19] fix: resolve TODOs and refactored --- modules/dagger/config.go | 143 +++++++++++++++++++++--------- modules/dagger/driver.go | 22 ++--- modules/dagger/schema/config.json | 14 ++- modules/flink/config.go | 9 ++ modules/flink/driver.go | 2 + modules/flink/driver_output.go | 1 + 6 files changed, 137 insertions(+), 54 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index 48f4db4..ede86ac 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -25,10 +25,12 @@ const keySinkInfluxMeasurementName = "SINK_INFLUX_MEASUREMENT_NAME" const keyRedisServer = "REDIS_SERVER" const SourceKafkaConsumerConfigAutoCommitEnable = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE" const SourceKafkaConsumerConfigAutoOffsetReset = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET" +const SourceKafkaConsumerConfigBootstrapServers = "SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS" const SinkTypeInflux = "INFLUX" const SinkTypeKafka = "KAFKA" const keySinkKafkaBrokers = "SINK_KAFKA_BROKERS" const keySinkKafkaStream = "SINK_KAFKA_STREAM" +const keySinkType = "SINK_TYPE" var ( //go:embed schema/config.json @@ -48,20 +50,23 @@ type Resources struct { } type Config struct { - Resources Resources `json:"resources,omitempty"` - FlinkName string `json:"flink_name,omitempty"` - DeploymentID string `json:"deployment_id,omitempty"` - Streams []Stream `json:"streams,omitempty"` - JobId string `json:"job_id,omitempty"` - Savepoint any `json:"savepoint,omitempty"` - EnvVariables map[string]string `json:"env_variables,omitempty"` - ChartValues *ChartValues `json:"chart_values,omitempty"` - Deleted bool `json:"deleted,omitempty"` - Namespace string `json:"namespace,omitempty"` - Replicas int `json:"replicas"` - SinkType string `json:"sink_type"` - PrometheusURL string `json:"prometheus_url"` - JarURI string `json:"jar_uri"` + Resources Resources `json:"resources,omitempty"` + Source []Source `json:"source,omitempty"` + Sink Sink `json:"sink,omitempty"` + EnvVariables map[string]string `json:"env_variables,omitempty"` + Replicas int `json:"replicas"` + SinkType string `json:"sink_type"` + Team string `json:"team"` + + FlinkName string `json:"flink_name,omitempty"` + DeploymentID string `json:"deployment_id,omitempty"` + JobId string `json:"job_id,omitempty"` + Savepoint any `json:"savepoint,omitempty"` + ChartValues *ChartValues `json:"chart_values,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Namespace string `json:"namespace,omitempty"` + PrometheusURL string `json:"prometheus_url,omitempty"` + JarURI string `json:"jar_uri,omitempty"` } type ChartValues struct { @@ -76,19 +81,65 @@ type SourceDetail struct { SourceType string `json:"SOURCE_TYPE"` } -type Stream struct { - InputSchemaTable string `json:"INPUT_SCHEMA_TABLE"` - SourceDetails []SourceDetail `json:"SOURCE_DETAILS"` +type SourceKafka struct { SourceKafkaConsumerConfigAutoCommitEnable string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE"` SourceKafkaConsumerConfigAutoOffsetReset string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET"` - InputSchemaProtoClass string `json:"INPUT_SCHEMA_PROTO_CLASS"` SourceKafkaTopicNames string `json:"SOURCE_KAFKA_TOPIC_NAMES"` SourceKafkaName string `json:"SOURCE_KAFKA_NAME"` - InputSchemaEventTimestampFieldIndex string `json:"INPUT_SCHEMA_EVENT_TIMESTAMP_FIELD_INDEX"` - SourceParquetFileDateRange interface{} `json:"SOURCE_PARQUET_FILE_DATE_RANGE"` - SourceParquetFilePaths interface{} `json:"SOURCE_PARQUET_FILE_PATHS"` SourceKafkaConsumerConfigGroupID string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_GROUP_ID"` SourceKafkaConsumerConfigBootstrapServers string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS"` + InputSchemaTable string `json:"INPUT_SCHEMA_TABLE"` + SourceDetails []SourceDetail `json:"SOURCE_DETAILS"` + InputSchemaProtoClass string `json:"INPUT_SCHEMA_PROTO_CLASS"` + InputSchemaEventTimestampFieldIndex string `json:"INPUT_SCHEMA_EVENT_TIMESTAMP_FIELD_INDEX"` +} + +type SourceParquet struct { + SourceParquetFileDateRange interface{} `json:"SOURCE_PARQUET_FILE_DATE_RANGE"` + SourceParquetFilePaths interface{} `json:"SOURCE_PARQUET_FILE_PATHS"` +} + +type Source struct { + SourceKafka + SourceParquet +} + +type SinkKafka struct { + SinkKafkaBrokers string `json:"SINK_KAFKA_BROKERS"` + SinkKafkaStream string `json:"SINK_KAFKA_STREAM"` + SinkKafkaTopic string `json:"SINK_KAFKA_TOPIC"` + SinkKafkaProtoMsg string `json:"SINK_KAFKA_PROTO_MESSAGE"` + SinkKafkaLingerMs string `json:"SINK_KAFKA_LINGER_MS"` +} + +type SinkInflux struct { + SinkInfluxDBName string `json:"SINK_INFLUX_DB_NAME"` + SinkInfluxMeasurementName string `json:"SINK_INFLUX_MEASUREMENT_NAME"` +} + +type SinkBigquery struct { + SinkBigqueryGoogleCloudProjectID string `json:"SINK_BIGQUERY_GOOGLE_CLOUD_PROJECT_ID"` + SinkBigqueryTableName string `json:"SINK_BIGQUERY_TABLE_NAME"` + SinkBigqueryDatasetLabels string `json:"SINK_BIGQUERY_DATASET_LABELS"` + SinkBigqueryTableLabels string `json:"SINK_BIGQUERY_TABLE_LABELS"` + SinkBigqueryDatasetName string `json:"SINK_BIGQUERY_DATASET_NAME"` + SinkBigqueryTablePartitioningEnable bool `json:"SINK_BIGQUERY_TABLE_PARTITIONING_ENABLE"` + SinkBigqueryTablePartitionKey string `json:"SINK_BIGQUERY_TABLE_PARTITION_KEY"` + SinkBigqueryRowInsertIDEnable bool `json:"SINK_BIGQUERY_ROW_INSERT_ID_ENABLE"` + SinkBigqueryClientReadTimeoutMs int `json:"SINK_BIGQUERY_CLIENT_READ_TIMEOUT_MS"` + SinkBigqueryClientConnectTimeoutMs int `json:"SINK_BIGQUERY_CLIENT_CONNECT_TIMEOUT_MS"` + SinkBigqueryTablePartitionExpiryMs int `json:"SINK_BIGQUERY_TABLE_PARTITION_EXPIRY_MS"` + SinkBigqueryDatasetLocation string `json:"SINK_BIGQUERY_DATASET_LOCATION"` + SinkBigqueryBatchSize int `json:"SINK_BIGQUERY_BATCH_SIZE"` + SinkBigqueryTableClusteringEnable bool `json:"SINK_BIGQUERY_TABLE_CLUSTERING_ENABLE"` + SinkBigqueryTableClusteringKeys string `json:"SINK_BIGQUERY_TABLE_CLUSTERING_KEYS"` + SinkErrorTypesForFailure string `json:"SINK_ERROR_TYPES_FOR_FAILURE"` +} + +type Sink struct { + SinkKafka + SinkInflux + SinkBigquery } func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverConf) (*Config, error) { @@ -99,11 +150,11 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo } //transformation #1 - streams := cfg.Streams + source := cfg.Source - for i := range streams { - if len(streams[i].SourceDetails) == 0 { - streams[i].SourceDetails = []SourceDetail{ + for i := range source { + if len(source[i].SourceDetails) == 0 { + source[i].SourceDetails = []SourceDetail{ { SourceName: "KAFKA_CONSUMER", SourceType: "UNBOUNDED", @@ -144,25 +195,28 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo cfg.EnvVariables[keySinkInfluxURL] = flinkOut.Influx.URL cfg.EnvVariables[keySinkInfluxPassword] = flinkOut.Influx.Password cfg.EnvVariables[keySinkInfluxUsername] = flinkOut.Influx.Username - //TODO: add sink influx db name - //TODO: check if SINK_INFLUX_MEASUREMENT_NAME and REDIS_SERVER needs modification + //SINK_INFLUX_DB_NAME is added by client + //SINK_INFLUX_MEASUREMENT_NAME is added by client + //REDIS_SERVER is skipped //transformation #8 - //Longbow related transformation skipped + //Longbow configs would be in base configs //transformation #9 and #11 - cfg.Streams = []Stream{} - for i := range streams { - if streams[i].SourceKafkaConsumerConfigGroupID == "" { - streams[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) + cfg.Source = []Source{} + for i := range source { + if source[i].SourceKafkaConsumerConfigGroupID == "" { + source[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) } - streams[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] - streams[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] - //TODO: add stream URL for key SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS - - cfg.Streams = append(cfg.Streams, Stream{ - SourceKafkaName: streams[i].SourceKafkaName, - SourceKafkaConsumerConfigGroupID: streams[i].SourceKafkaConsumerConfigGroupID, + source[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] + source[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] + source[i].SourceKafkaConsumerConfigBootstrapServers = dc.EnvVariables[SourceKafkaConsumerConfigBootstrapServers] + + cfg.Source = append(cfg.Source, Source{ + SourceKafka: SourceKafka{ + SourceKafkaName: source[i].SourceKafkaName, + SourceKafkaConsumerConfigGroupID: source[i].SourceKafkaConsumerConfigGroupID, + }, }) } @@ -171,18 +225,23 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo //do we need to check this? //transformation #12 - cfg.EnvVariables[keyStreams] = string(mustMarshalJSON(streams)) + cfg.EnvVariables[keyStreams] = string(mustMarshalJSON(source)) //transformation #13 + cfg.EnvVariables[keySinkType] = cfg.SinkType if cfg.SinkType == SinkTypeKafka { - cfg.EnvVariables[keySinkKafkaStream] = flinkOut.SinkKafkaStream - //TODO cfg.EnvVariables[keySinkKafkaBrokers] = stream URL + cfg.EnvVariables[keySinkKafkaStream] = cfg.Sink.SinkKafka.SinkKafkaStream + cfg.EnvVariables[keySinkKafkaBrokers] = cfg.Sink.SinkKafka.SinkKafkaBrokers + } else if cfg.SinkType == SinkTypeInflux { + cfg.EnvVariables[keySinkInfluxDBName] = cfg.Sink.SinkInflux.SinkInfluxDBName + cfg.EnvVariables[keySinkInfluxMeasurementName] = cfg.Sink.SinkInflux.SinkInfluxMeasurementName } //transformation #14 cfg.Resources = mergeResources(dc.Resources, cfg.Resources) cfg.PrometheusURL = flinkOut.PrometheusURL + cfg.FlinkName = flinkOut.FlinkName if cfg.Replicas <= 0 { cfg.Replicas = 1 diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go index 70c5e15..c52eb2f 100644 --- a/modules/dagger/driver.go +++ b/modules/dagger/driver.go @@ -3,8 +3,11 @@ package dagger import ( "bytes" "context" + "encoding/base64" "encoding/json" + "fmt" "html/template" + "strings" "time" "github.com/goto/entropy/core/module" @@ -163,13 +166,11 @@ func (dd *daggerDriver) getHelmRelease(res resource.Resource, conf Config, imageRepository = conf.ChartValues.ImageRepository } - _, err = json.Marshal(conf.EnvVariables) - if err != nil { - return nil, errors.ErrInternal.WithMsgf("failed to marshal env variables").WithCausef(err.Error()) + var programArgs []string + for key, value := range conf.EnvVariables { + programArgs = append(programArgs, "--"+key, fmt.Sprintf("%v", value)) } - - //encodedEnvVars := base64.StdEncoding.EncodeToString(envVarsJSON) - //programArgs := encodedEnvVars + encodedProgramArgs := base64.StdEncoding.EncodeToString([]byte(strings.Join(programArgs, ""))) rc.Values = map[string]any{ labelsConfKey: modules.CloneAndMergeMaps(deploymentLabels, entropyLabels), @@ -180,8 +181,8 @@ func (dd *daggerDriver) getHelmRelease(res resource.Resource, conf Config, }, "projectID": res.Project, "name": res.Name, - "team": res.Labels["team"], //TODO: improve handling this case - "flink_name": "g-pilotdata-gl-playground-operator-v2", //TODO: use this from flink resource + "team": conf.Team, + "flink_name": conf.FlinkName, "prometheus_url": conf.PrometheusURL, "resources": map[string]any{ "jobmanager": map[string]any{ @@ -193,9 +194,8 @@ func (dd *daggerDriver) getHelmRelease(res resource.Resource, conf Config, "memory": conf.Resources.JobManager.Memory, }, }, - "jarURI": conf.JarURI, - //TODO: build programArgs from EnvVariables - "programArgs": []string{"--encodedArgs", "WyItLVNPVVJDRV9LQUZLQV9DT05TVU1FX0xBUkdFX01FU1NBR0VfRU5BQkxFIixmYWxzZSwiLS1FTkFCTEVfU1RFTkNJTF9VUkwiLHRydWUsIi0tRkxJTktfSk9CX0lEIiwiZy1waWxvdGRhdGEtZ2wtdGVzdC1kb2NrZXItZXJyb3ItZGFnZ2VyIiwiLS1TSU5LX0lORkxVWF9CQVRDSF9TSVpFIiwxMDAsIi0tU0lOS19JTkZMVVhfREJfTkFNRSIsIkRBR0dFUlNfUExBWUdST1VORCIsIi0tU0lOS19JTkZMVVhfRkxVU0hfRFVSQVRJT05fTVMiLDEwMDAsIi0tU0lOS19JTkZMVVhfTUVBU1VSRU1FTlRfTkFNRSIsInRlc3QtZG9ja2VyLWVycm9yIiwiLS1TSU5LX0lORkxVWF9QQVNTV09SRCIsInJvb3QiLCItLVNJTktfSU5GTFVYX1JFVEVOVElPTl9QT0xJQ1kiLCJhdXRvZ2VuIiwiLS1TSU5LX0lORkxVWF9VUkwiLCJodHRwOi8vMTAuODQuNTIuODk6ODA4NiIsIi0tU0lOS19JTkZMVVhfVVNFUk5BTUUiLCJyb290IiwiLS1QUk9DRVNTT1JfTE9OR0JPV19HQ1BfSU5TVEFOQ0VfSUQiLCJ0ZXN0IiwiLS1QUk9DRVNTT1JfTE9OR0JPV19HQ1BfUFJPSkVDVF9JRCIsInRlc3QiLCItLVNJTktfS0FGS0FfQlJPS0VSUyIsIiIsIi0tU0lOS19LQUZLQV9UT1BJQyIsIiIsIi0tT1VUUFVUX1BST1RPX0NMQVNTX1BSRUZJWCIsIiIsIi0tU0lOS19LQUZLQV9QUk9UT19LRVkiLCIiLCItLVNJTktfS0FGS0FfUFJPVE9fTUVTU0FHRSIsIiIsIi0tU0lOS19LQUZLQV9TVFJFQU0iLCIiLCItLUZMSU5LX1BBUkFMTEVMSVNNIiwxLCItLVBPUlRBTF9WRVJTSU9OIiwiMiIsIi0tUFJPQ0VTU09SX1BSRVBST0NFU1NPUl9DT05GSUciLCIiLCItLVBST0NFU1NPUl9QUkVQUk9DRVNTT1JfRU5BQkxFIixmYWxzZSwiLS1QUk9DRVNTT1JfUE9TVFBST0NFU1NPUl9DT05GSUciLCIiLCItLVBST0NFU1NPUl9QT1NUUFJPQ0VTU09SX0VOQUJMRSIsZmFsc2UsIi0tU0lOS19LQUZLQV9QUk9EVUNFX0xBUkdFX01FU1NBR0VfRU5BQkxFIixmYWxzZSwiLS1SRURJU19TRVJWRVIiLCIiLCItLUZMSU5LX1JPV1RJTUVfQVRUUklCVVRFX05BTUUiLCJyb3d0aW1lIiwiLS1TSU5LX1RZUEUiLCJpbmZsdXgiLCItLUZMSU5LX1NRTF9RVUVSWSIsIlNFTEVDVCAqIEZST00gZGF0YV9zdHJlYW1zXzAiLCItLVNDSEVNQV9SRUdJU1RSWV9TVEVOQ0lMX0VOQUJMRSIsdHJ1ZSwiLS1TQ0hFTUFfUkVHSVNUUllfU1RFTkNJTF9VUkxTIiwiaHR0cDovL2ctZ29kYXRhLXN5c3RlbXMtc3RlbmNpbC12MWJldGExLWluZ3Jlc3MuZ29sYWJzLmlvL3YxYmV0YTEvbmFtZXNwYWNlcy9nb2play9zY2hlbWFzL2VzYi1sb2ctZW50aXRpZXMvdmVyc2lvbnMvOCIsIi0tU1RSRUFNUyIsIlt7XCJJTlBVVF9TQ0hFTUFfRVZFTlRfVElNRVNUQU1QX0ZJRUxEX0lOREVYXCI6XCI1XCIsXCJJTlBVVF9TQ0hFTUFfUFJPVE9fQ0xBU1NcIjpcImdvamVrLmVzYi5ib29raW5nLkJvb2tpbmdMb2dNZXNzYWdlXCIsXCJJTlBVVF9TQ0hFTUFfVEFCTEVcIjpcImRhdGFfc3RyZWFtc18wXCIsXCJTT1VSQ0VfREVUQUlMU1wiOlt7XCJTT1VSQ0VfTkFNRVwiOlwiS0FGS0FfQ09OU1VNRVJcIixcIlNPVVJDRV9UWVBFXCI6XCJVTkJPVU5ERURcIn1dLFwiU09VUkNFX0tBRktBX0NPTlNVTUVSX0NPTkZJR19BVVRPX0NPTU1JVF9FTkFCTEVcIjpcImZhbHNlXCIsXCJTT1VSQ0VfS0FGS0FfQ09OU1VNRVJfQ09ORklHX0FVVE9fT0ZGU0VUX1JFU0VUXCI6XCJsYXRlc3RcIixcIlNPVVJDRV9LQUZLQV9DT05TVU1FUl9DT05GSUdfQk9PVFNUUkFQX1NFUlZFUlNcIjpcImctcGlsb3RkYXRhLWdsLW1haW5zdHJlYW0ta2Fma2EuZ29sYWJzLmlvOjY2NjhcIixcIlNPVVJDRV9LQUZLQV9DT05TVU1FUl9DT05GSUdfR1JPVVBfSURcIjpcImctcGlsb3RkYXRhLWdsLWRhZ2dlci10ZXN0LTAxLWRhZ2dlci0wMDAxXCIsXCJTT1VSQ0VfS0FGS0FfTkFNRVwiOlwiZy1waWxvdGRhdGEtZ2wtbWFpbnN0cmVhbVwiLFwiU09VUkNFX0tBRktBX1RPUElDX05BTUVTXCI6XCJhZ2dyZWdhdGVkZGVtYW5kLXByaWNpbmd0ZXN0XCIsXCJTT1VSQ0VfUEFSUVVFVF9GSUxFX0RBVEVfUkFOR0VcIjpudWxsLFwiU09VUkNFX1BBUlFVRVRfRklMRV9QQVRIU1wiOm51bGx9XSIsIi0tRkxJTktfV0FURVJNQVJLX0RFTEFZX01TIiwxMDAwLCItLUZMSU5LX1dBVEVSTUFSS19JTlRFUlZBTF9NUyIsNjAwMDAsIi0tVURGX0RBUlRfR0NTX0JVQ0tFVF9JRCIsInAtZ29kYXRhLWRhZ2dlcnMtZGFydHMtc3RvcmFnZSIsIi0tVURGX0RBUlRfR0NTX1BST0pFQ1RfSUQiLCJnb2RhdGEtcHJvZHVjdGlvbiIsIi0tRlVOQ1RJT05fRkFDVE9SWV9DTEFTU0VTIiwiY29tLmdvdG9jb21wYW55LmRhZ2dlci5mdW5jdGlvbnMudWRmcy5mYWN0b3JpZXMuRnVuY3Rpb25GYWN0b3J5LGNvbS5nb2play5kZS5mbHVkLnVkZnMuZmFjdG9yaWVzLkZ1bmN0aW9uRmFjdG9yeSIsIi0tUFlUSE9OX1VERl9FTkFCTEUiLGZhbHNlLCItLVBZVEhPTl9VREZfQ09ORklHIiwie1wiUFlUSE9OX0ZJTEVTXCI6IFwiZ3M6Ly9nb2RhdGEtZGFnZ2VyL3B5dGhvbi9tYXN0ZXIvbGF0ZXN0L3B5dGhvbl91ZGZzLnppcFwiLFwiUFlUSE9OX1JFUVVJUkVNRU5UU1wiOiBcImdzOi8vZ29kYXRhLWRhZ2dlci9weXRob24vbWFzdGVyL2xhdGVzdC9yZXF1aXJlbWVudHMudHh0XCIsXCJQWVRIT05fQVJDSElWRVNcIjogXCJnczovL2dvZGF0YS1kYWdnZXIvcHl0aG9uL21hc3Rlci9sYXRlc3QvZGF0YS56aXAjZGF0YVwiLFwiUFlUSE9OX0ZOX0VYRUNVVElPTl9BUlJPV19CQVRDSF9TSVpFXCI6IFwiMTAwMDBcIixcIlBZVEhPTl9GTl9FWEVDVVRJT05fQlVORExFX1NJWkVcIjogXCIxMDAwMDBcIixcIlBZVEhPTl9GTl9FWEVDVVRJT05fQlVORExFX1RJTUVcIjogXCIxMDAwXCJ9IiwiLS1TSU5LX0JJR1FVRVJZX0dPT0dMRV9DTE9VRF9QUk9KRUNUX0lEIiwiIiwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX05BTUUiLCIiLCItLVNJTktfQklHUVVFUllfREFUQVNFVF9MQUJFTFMiLCIiLCItLVNJTktfQklHUVVFUllfVEFCTEVfTEFCRUxTIiwiIiwiLS1TSU5LX0JJR1FVRVJZX0RBVEFTRVRfTkFNRSIsIiIsIi0tU0lOS19CSUdRVUVSWV9DUkVERU5USUFMX1BBVEgiLCIvdmFyL3NlY3JldHMvZ29vZ2xlL2djcF9rZXkuanNvbiIsIi0tU0lOS19CSUdRVUVSWV9UQUJMRV9QQVJUSVRJT05JTkdfRU5BQkxFIixmYWxzZSwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX1BBUlRJVElPTl9LRVkiLCIiLCItLVNJTktfQklHUVVFUllfUk9XX0lOU0VSVF9JRF9FTkFCTEUiLGZhbHNlLCItLVNJTktfQklHUVVFUllfQ0xJRU5UX1JFQURfVElNRU9VVF9NUyIsLTEsIi0tU0lOS19CSUdRVUVSWV9DTElFTlRfQ09OTkVDVF9USU1FT1VUX01TIiwtMSwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX1BBUlRJVElPTl9FWFBJUllfTVMiLC0xLCItLVNJTktfQklHUVVFUllfREFUQVNFVF9MT0NBVElPTiIsImFzaWEtc291dGhlYXN0MSIsIi0tU0lOS19CSUdRVUVSWV9NRVRBREFUQV9OQU1FU1BBQ0UiLCIiLCItLVNJTktfQklHUVVFUllfQUREX01FVEFEQVRBX0VOQUJMRUQiLGZhbHNlLCItLVNJTktfQklHUVVFUllfTUVUQURBVEFfQ09MVU1OU19UWVBFUyIsIiIsIi0tU0lOS19NRVRSSUNTX0FQUExJQ0FUSU9OX1BSRUZJWCIsImRhZ2dlcl8iLCItLVNJTktfQklHUVVFUllfQkFUQ0hfU0laRSIsIiIsIi0tU0lOS19DT05ORUNUT1JfU0NIRU1BX1BST1RPX01FU1NBR0VfQ0xBU1MiLCIiLCItLVNJTktfQ09OTkVDVE9SX1NDSEVNQV9QUk9UT19LRVlfQ0xBU1MiLCIiLCItLVNJTktfQ09OTkVDVE9SX1NDSEVNQV9EQVRBX1RZUEUiLCJQUk9UT0JVRiIsIi0tU0lOS19DT05ORUNUT1JfU0NIRU1BX01FU1NBR0VfTU9ERSIsIkxPR19NRVNTQUdFIiwiLS1TSU5LX0NPTk5FQ1RPUl9TQ0hFTUFfUFJPVE9fQUxMT1dfVU5LTk9XTl9GSUVMRFNfRU5BQkxFIixmYWxzZSwiLS1TSU5LX0VSUk9SX1RZUEVTX0ZPUl9GQUlMVVJFIiwiIiwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX0NMVVNURVJJTkdfRU5BQkxFIixmYWxzZSwiLS1TSU5LX0JJR1FVRVJZX1RBQkxFX0NMVVNURVJJTkdfS0VZUyIsIiIsIi0tU0NIRU1BX1JFR0lTVFJZX1NURU5DSUxfQ0FDSEVfQVVUT19SRUZSRVNIIixmYWxzZSwiLS1TQ0hFTUFfUkVHSVNUUllfU1RFTkNJTF9SRUZSRVNIX1NUUkFURUdZIiwiTE9OR19QT0xMSU5HIiwiLS1TQ0hFTUFfUkVHSVNUUllfU1RFTkNJTF9DQUNIRV9UVExfTVMiLCI5MDAwMDAiLCItLVNJTktfS0FGS0FfTElOR0VSX01TIiwiMCIsIi0tU0NIRU1BX1JFR0lTVFJZX1NURU5DSUxfRkVUQ0hfUkVUUklFUyIsIjQiLCItLVNDSEVNQV9SRUdJU1RSWV9TVEVOQ0lMX0ZFVENIX0JBQ0tPRkZfTUlOX01TIiwiNjAwMDAiLCItLVNDSEVNQV9SRUdJU1RSWV9TVEVOQ0lMX0ZFVENIX1RJTUVPVVRfTVMiLCIxMDAwIl0="}, + "jarURI": conf.JarURI, + "programArgs": append([]string{"--encodedArgs"}, encodedProgramArgs), "state": "running", "namespace": conf.Namespace, } diff --git a/modules/dagger/schema/config.json b/modules/dagger/schema/config.json index 453755b..3effabd 100644 --- a/modules/dagger/schema/config.json +++ b/modules/dagger/schema/config.json @@ -4,7 +4,11 @@ "type": "object", "required": [ "replicas", - "env_variables" + "env_variables", + "team", + "source", + "sink", + "sink_type" ], "properties": { "replicas": { @@ -15,6 +19,14 @@ "deployment_id": { "type": "string" }, + "sink_type": { + "type": "string", + "enum": [ + "INFLUX", + "KAFKA", + "BIGQUERY" + ] + }, "env_variables": { "type": "object", "additionalProperties": true, diff --git a/modules/flink/config.go b/modules/flink/config.go index 50401f1..6ea778a 100644 --- a/modules/flink/config.go +++ b/modules/flink/config.go @@ -27,6 +27,7 @@ type Config struct { Influx Influx `json:"influx,omitempty"` SinkKafkaStream string `json:"sink_kafka_stream,omitempty"` PrometheusURL string `json:"prometheus_url,omitempty"` + FlinkName string `json:"flink_name,omitempty"` } func readConfig(_ resource.Resource, confJSON json.RawMessage, dc driverConf) (*Config, error) { @@ -49,5 +50,13 @@ func readConfig(_ resource.Resource, confJSON json.RawMessage, dc driverConf) (* cfg.KubeNamespace = dc.KubeNamespace } + if cfg.PrometheusURL == "" { + cfg.PrometheusURL = dc.PrometheusURL + } + + if cfg.FlinkName == "" { + cfg.FlinkName = dc.FlinkName + } + return &cfg, nil } diff --git a/modules/flink/driver.go b/modules/flink/driver.go index 192df3d..9fbb0b2 100644 --- a/modules/flink/driver.go +++ b/modules/flink/driver.go @@ -17,6 +17,7 @@ type driverConf struct { SinkKafkaStream string `json:"sink_kafka_stream,omitempty"` KubeNamespace string `json:"kube_namespace,omitempty"` PrometheusURL string `json:"prometheus_url,omitempty"` + FlinkName string `json:"flink_name,omitempty"` } type Output struct { @@ -25,6 +26,7 @@ type Output struct { Influx Influx `json:"influx,omitempty"` SinkKafkaStream string `json:"sink_kafka_stream,omitempty"` PrometheusURL string `json:"prometheus_url,omitempty"` + FlinkName string `json:"flink_name,omitempty"` } func readOutputData(exr module.ExpandedResource) (*Output, error) { diff --git a/modules/flink/driver_output.go b/modules/flink/driver_output.go index 01a58b0..58596a9 100644 --- a/modules/flink/driver_output.go +++ b/modules/flink/driver_output.go @@ -34,6 +34,7 @@ func (fd *flinkDriver) Output(ctx context.Context, exr module.ExpandedResource) output.KubeNamespace = conf.KubeNamespace output.SinkKafkaStream = conf.SinkKafkaStream output.PrometheusURL = conf.PrometheusURL + output.FlinkName = conf.FlinkName return modules.MustJSON(output), nil } From b223663699dc202633e71f0ec47e023210707357 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 18 Sep 2024 17:55:55 +0530 Subject: [PATCH 08/19] fix: source sink base handling --- modules/dagger/config.go | 145 ++++++++++++++++++++++++++------------- modules/dagger/driver.go | 12 +++- pkg/helm/helm.go | 3 - 3 files changed, 106 insertions(+), 54 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index ede86ac..55bafa8 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -28,9 +28,32 @@ const SourceKafkaConsumerConfigAutoOffsetReset = "SOURCE_KAFKA_CONSUMER_CONFIG_A const SourceKafkaConsumerConfigBootstrapServers = "SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS" const SinkTypeInflux = "INFLUX" const SinkTypeKafka = "KAFKA" +const SinkTypeBigquery = "BIGQUERY" const keySinkKafkaBrokers = "SINK_KAFKA_BROKERS" const keySinkKafkaStream = "SINK_KAFKA_STREAM" const keySinkType = "SINK_TYPE" +const keySinkKafkaProtoMsg = "SINK_KAFKA_PROTO_MESSAGE" +const keySinkKafkaTopic = "SINK_KAFKA_TOPIC" +const keySinkKafkaKey = "SINK_KAFKA_PROTO_KEY" +const keySinkKafkaLingerMs = "SINK_KAFKA_LINGER_MS" +const ( + keySinkBigqueryGoogleCloudProjectID = "SINK_BIGQUERY_GOOGLE_CLOUD_PROJECT_ID" + keySinkBigqueryDatasetName = "SINK_BIGQUERY_DATASET_NAME" + keySinkBigqueryTableName = "SINK_BIGQUERY_TABLE_NAME" + keySinkBigqueryDatasetLabels = "SINK_BIGQUERY_DATASET_LABELS" + keySinkBigqueryTableLabels = "SINK_BIGQUERY_TABLE_LABELS" + keySinkBigqueryTablePartitioningEnable = "SINK_BIGQUERY_TABLE_PARTITIONING_ENABLE" + keySinkBigqueryTableClusteringEnable = "SINK_BIGQUERY_TABLE_CLUSTERING_ENABLE" + keySinkBigqueryBatchSize = "SINK_BIGQUERY_BATCH_SIZE" + keySinkBigqueryTablePartitionKey = "SINK_BIGQUERY_TABLE_PARTITION_KEY" + keySinkBigqueryRowInsertIDEnable = "SINK_BIGQUERY_ROW_INSERT_ID_ENABLE" + keySinkBigqueryClientReadTimeoutMs = "SINK_BIGQUERY_CLIENT_READ_TIMEOUT_MS" + keySinkBigqueryClientConnectTimeoutMs = "SINK_BIGQUERY_CLIENT_CONNECT_TIMEOUT_MS" + keySinkBigqueryTablePartitionExpiryMs = "SINK_BIGQUERY_TABLE_PARTITION_EXPIRY_MS" + keySinkBigqueryDatasetLocation = "SINK_BIGQUERY_DATASET_LOCATION" + keySinkErrorTypesForFailure = "SINK_ERROR_TYPES_FOR_FAILURE" + keySinkBigqueryTableClusteringKeys = "SINK_BIGQUERY_TABLE_CLUSTERING_KEYS" +) var ( //go:embed schema/config.json @@ -82,24 +105,24 @@ type SourceDetail struct { } type SourceKafka struct { - SourceKafkaConsumerConfigAutoCommitEnable string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE"` - SourceKafkaConsumerConfigAutoOffsetReset string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET"` - SourceKafkaTopicNames string `json:"SOURCE_KAFKA_TOPIC_NAMES"` - SourceKafkaName string `json:"SOURCE_KAFKA_NAME"` - SourceKafkaConsumerConfigGroupID string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_GROUP_ID"` - SourceKafkaConsumerConfigBootstrapServers string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS"` - InputSchemaTable string `json:"INPUT_SCHEMA_TABLE"` - SourceDetails []SourceDetail `json:"SOURCE_DETAILS"` - InputSchemaProtoClass string `json:"INPUT_SCHEMA_PROTO_CLASS"` - InputSchemaEventTimestampFieldIndex string `json:"INPUT_SCHEMA_EVENT_TIMESTAMP_FIELD_INDEX"` + SourceKafkaConsumerConfigAutoCommitEnable string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE"` + SourceKafkaConsumerConfigAutoOffsetReset string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET"` + SourceKafkaTopicNames string `json:"SOURCE_KAFKA_TOPIC_NAMES"` + SourceKafkaName string `json:"SOURCE_KAFKA_NAME"` + SourceKafkaConsumerConfigGroupID string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_GROUP_ID"` + SourceKafkaConsumerConfigBootstrapServers string `json:"SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS"` } type SourceParquet struct { SourceParquetFileDateRange interface{} `json:"SOURCE_PARQUET_FILE_DATE_RANGE"` - SourceParquetFilePaths interface{} `json:"SOURCE_PARQUET_FILE_PATHS"` + SourceParquetFilePaths []string `json:"SOURCE_PARQUET_FILE_PATHS"` } type Source struct { + InputSchemaProtoClass string `json:"INPUT_SCHEMA_PROTO_CLASS"` + InputSchemaEventTimestampFieldIndex string `json:"INPUT_SCHEMA_EVENT_TIMESTAMP_FIELD_INDEX"` + SourceDetails []SourceDetail `json:"SOURCE_DETAILS"` + InputSchemaTable string `json:"INPUT_SCHEMA_TABLE"` SourceKafka SourceParquet } @@ -110,6 +133,7 @@ type SinkKafka struct { SinkKafkaTopic string `json:"SINK_KAFKA_TOPIC"` SinkKafkaProtoMsg string `json:"SINK_KAFKA_PROTO_MESSAGE"` SinkKafkaLingerMs string `json:"SINK_KAFKA_LINGER_MS"` + SinkKafkaProtoKey string `json:"SINK_KAFKA_PROTO_KEY"` } type SinkInflux struct { @@ -123,15 +147,15 @@ type SinkBigquery struct { SinkBigqueryDatasetLabels string `json:"SINK_BIGQUERY_DATASET_LABELS"` SinkBigqueryTableLabels string `json:"SINK_BIGQUERY_TABLE_LABELS"` SinkBigqueryDatasetName string `json:"SINK_BIGQUERY_DATASET_NAME"` - SinkBigqueryTablePartitioningEnable bool `json:"SINK_BIGQUERY_TABLE_PARTITIONING_ENABLE"` + SinkBigqueryTablePartitioningEnable string `json:"SINK_BIGQUERY_TABLE_PARTITIONING_ENABLE"` SinkBigqueryTablePartitionKey string `json:"SINK_BIGQUERY_TABLE_PARTITION_KEY"` - SinkBigqueryRowInsertIDEnable bool `json:"SINK_BIGQUERY_ROW_INSERT_ID_ENABLE"` - SinkBigqueryClientReadTimeoutMs int `json:"SINK_BIGQUERY_CLIENT_READ_TIMEOUT_MS"` - SinkBigqueryClientConnectTimeoutMs int `json:"SINK_BIGQUERY_CLIENT_CONNECT_TIMEOUT_MS"` - SinkBigqueryTablePartitionExpiryMs int `json:"SINK_BIGQUERY_TABLE_PARTITION_EXPIRY_MS"` + SinkBigqueryRowInsertIDEnable string `json:"SINK_BIGQUERY_ROW_INSERT_ID_ENABLE"` + SinkBigqueryClientReadTimeoutMs string `json:"SINK_BIGQUERY_CLIENT_READ_TIMEOUT_MS"` + SinkBigqueryClientConnectTimeoutMs string `json:"SINK_BIGQUERY_CLIENT_CONNECT_TIMEOUT_MS"` + SinkBigqueryTablePartitionExpiryMs string `json:"SINK_BIGQUERY_TABLE_PARTITION_EXPIRY_MS"` SinkBigqueryDatasetLocation string `json:"SINK_BIGQUERY_DATASET_LOCATION"` - SinkBigqueryBatchSize int `json:"SINK_BIGQUERY_BATCH_SIZE"` - SinkBigqueryTableClusteringEnable bool `json:"SINK_BIGQUERY_TABLE_CLUSTERING_ENABLE"` + SinkBigqueryBatchSize string `json:"SINK_BIGQUERY_BATCH_SIZE"` + SinkBigqueryTableClusteringEnable string `json:"SINK_BIGQUERY_TABLE_CLUSTERING_ENABLE"` SinkBigqueryTableClusteringKeys string `json:"SINK_BIGQUERY_TABLE_CLUSTERING_KEYS"` SinkErrorTypesForFailure string `json:"SINK_ERROR_TYPES_FOR_FAILURE"` } @@ -149,18 +173,41 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo return nil, errors.ErrInvalid.WithMsgf("invalid config json").WithCausef(err.Error()) } - //transformation #1 - source := cfg.Source - - for i := range source { - if len(source[i].SourceDetails) == 0 { - source[i].SourceDetails = []SourceDetail{ - { - SourceName: "KAFKA_CONSUMER", - SourceType: "UNBOUNDED", - }, + //transformation #9 and #11 + if cfg.EnvVariables[keyStreams] == "" { + //transformation #1 + source := cfg.Source + + cfg.Source = []Source{} + for i := range source { + if source[i].SourceParquet.SourceParquetFilePaths != nil && len(source[i].SourceParquet.SourceParquetFilePaths) > 0 { + //source is parquete + //do nothing + cfg.Source = []Source{ + { + SourceParquet: SourceParquet{}, + }, + } + continue + } + if source[i].SourceKafkaConsumerConfigGroupID == "" { + source[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) } + source[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] + source[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] + source[i].SourceKafkaConsumerConfigBootstrapServers = dc.EnvVariables[SourceKafkaConsumerConfigBootstrapServers] + + cfg.Source = append(cfg.Source, Source{ + SourceKafka: SourceKafka{ + SourceKafkaName: source[i].SourceKafkaName, + SourceKafkaConsumerConfigGroupID: source[i].SourceKafkaConsumerConfigGroupID, + }, + }) } + + //transformation #12 + cfg.EnvVariables[keyStreams] = string(mustMarshalJSON(source)) + fmt.Printf("cfg.EnvVariables[keyStreams]: %v\n", cfg.EnvVariables[keyStreams]) } //transformation #2 @@ -202,39 +249,39 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo //transformation #8 //Longbow configs would be in base configs - //transformation #9 and #11 - cfg.Source = []Source{} - for i := range source { - if source[i].SourceKafkaConsumerConfigGroupID == "" { - source[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) - } - source[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] - source[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] - source[i].SourceKafkaConsumerConfigBootstrapServers = dc.EnvVariables[SourceKafkaConsumerConfigBootstrapServers] - - cfg.Source = append(cfg.Source, Source{ - SourceKafka: SourceKafka{ - SourceKafkaName: source[i].SourceKafkaName, - SourceKafkaConsumerConfigGroupID: source[i].SourceKafkaConsumerConfigGroupID, - }, - }) - } - //transformation #10 //this shall check if the project of the conf.EnvVars.STREAMS is same as that of the corresponding flink //do we need to check this? - //transformation #12 - cfg.EnvVariables[keyStreams] = string(mustMarshalJSON(source)) - //transformation #13 cfg.EnvVariables[keySinkType] = cfg.SinkType if cfg.SinkType == SinkTypeKafka { cfg.EnvVariables[keySinkKafkaStream] = cfg.Sink.SinkKafka.SinkKafkaStream cfg.EnvVariables[keySinkKafkaBrokers] = cfg.Sink.SinkKafka.SinkKafkaBrokers + cfg.EnvVariables[keySinkKafkaProtoMsg] = cfg.Sink.SinkKafka.SinkKafkaProtoMsg + cfg.EnvVariables[keySinkKafkaTopic] = cfg.Sink.SinkKafka.SinkKafkaTopic + cfg.EnvVariables[keySinkKafkaKey] = cfg.Sink.SinkKafka.SinkKafkaProtoKey + cfg.EnvVariables[keySinkKafkaLingerMs] = cfg.Sink.SinkKafka.SinkKafkaLingerMs } else if cfg.SinkType == SinkTypeInflux { cfg.EnvVariables[keySinkInfluxDBName] = cfg.Sink.SinkInflux.SinkInfluxDBName cfg.EnvVariables[keySinkInfluxMeasurementName] = cfg.Sink.SinkInflux.SinkInfluxMeasurementName + } else if cfg.SinkType == SinkTypeBigquery { + cfg.EnvVariables[keySinkBigqueryGoogleCloudProjectID] = cfg.Sink.SinkBigquery.SinkBigqueryGoogleCloudProjectID + cfg.EnvVariables[keySinkBigqueryDatasetName] = cfg.Sink.SinkBigquery.SinkBigqueryDatasetName + cfg.EnvVariables[keySinkBigqueryTableName] = cfg.Sink.SinkBigquery.SinkBigqueryTableName + cfg.EnvVariables[keySinkBigqueryDatasetLabels] = cfg.Sink.SinkBigquery.SinkBigqueryDatasetLabels + cfg.EnvVariables[keySinkBigqueryTableLabels] = cfg.Sink.SinkBigquery.SinkBigqueryTableLabels + cfg.EnvVariables[keySinkBigqueryTablePartitioningEnable] = cfg.Sink.SinkBigquery.SinkBigqueryTablePartitioningEnable + cfg.EnvVariables[keySinkBigqueryTablePartitionKey] = cfg.Sink.SinkBigquery.SinkBigqueryTablePartitionKey + cfg.EnvVariables[keySinkBigqueryRowInsertIDEnable] = cfg.Sink.SinkBigquery.SinkBigqueryRowInsertIDEnable + cfg.EnvVariables[keySinkBigqueryClientReadTimeoutMs] = cfg.Sink.SinkBigquery.SinkBigqueryClientReadTimeoutMs + cfg.EnvVariables[keySinkBigqueryClientConnectTimeoutMs] = cfg.Sink.SinkBigquery.SinkBigqueryClientConnectTimeoutMs + cfg.EnvVariables[keySinkBigqueryTablePartitionExpiryMs] = cfg.Sink.SinkBigquery.SinkBigqueryTablePartitionExpiryMs + cfg.EnvVariables[keySinkBigqueryDatasetLocation] = cfg.Sink.SinkBigquery.SinkBigqueryDatasetLocation + cfg.EnvVariables[keySinkBigqueryBatchSize] = cfg.Sink.SinkBigquery.SinkBigqueryBatchSize + cfg.EnvVariables[keySinkBigqueryTableClusteringEnable] = cfg.Sink.SinkBigquery.SinkBigqueryTableClusteringEnable + cfg.EnvVariables[keySinkBigqueryTableClusteringKeys] = cfg.Sink.SinkBigquery.SinkBigqueryTableClusteringKeys + cfg.EnvVariables[keySinkErrorTypesForFailure] = cfg.Sink.SinkBigquery.SinkErrorTypesForFailure } //transformation #14 diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go index c52eb2f..53ba11c 100644 --- a/modules/dagger/driver.go +++ b/modules/dagger/driver.go @@ -168,9 +168,17 @@ func (dd *daggerDriver) getHelmRelease(res resource.Resource, conf Config, var programArgs []string for key, value := range conf.EnvVariables { - programArgs = append(programArgs, "--"+key, fmt.Sprintf("%v", value)) + // Check if the value is a JSON object and escape quotes if necessary + if json.Valid([]byte(value)) { + value = strings.ReplaceAll(value, `"`, `\"`) + } + programArgs = append(programArgs, fmt.Sprintf("\"%s\"", "--"+key), fmt.Sprintf("\"%v\"", value)) } - encodedProgramArgs := base64.StdEncoding.EncodeToString([]byte(strings.Join(programArgs, ""))) + + //fmt.Printf("programArgs: %v\n", programArgs) + formatted := fmt.Sprintf("[%s]", strings.Join(programArgs, ",")) + //fmt.Printf("formatted: %v\n", formatted) + encodedProgramArgs := base64.StdEncoding.EncodeToString([]byte(formatted)) rc.Values = map[string]any{ labelsConfKey: modules.CloneAndMergeMaps(deploymentLabels, entropyLabels), diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 2744129..c06b59f 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -92,9 +92,6 @@ func (p *Client) doCreate(actionConfig *action.Configuration, config *ReleaseCon return nil, errors.ErrInternal.WithMsgf("create-release failed").WithCausef(err.Error()) } - //TODO: remove this - fmt.Println("Release manifest\n", rel) - return &Result{ Config: config, Release: rel, From 593c69ab1d961c8cbfd0bfea07af47061f1f911a Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Thu, 19 Sep 2024 16:42:57 +0530 Subject: [PATCH 09/19] feat: Output to have CR details --- modules/dagger/config.go | 33 +++++++++--------- modules/dagger/driver.go | 7 ++-- modules/dagger/driver_output.go | 12 ++++++- modules/dagger/driver_plan.go | 3 +- modules/dagger/module.go | 59 +++++++++++++++++++++++++++++++-- pkg/kube/client.go | 33 ++++++++++++++++++ 6 files changed, 123 insertions(+), 24 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index 55bafa8..6b16a1c 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -73,23 +73,21 @@ type Resources struct { } type Config struct { - Resources Resources `json:"resources,omitempty"` - Source []Source `json:"source,omitempty"` - Sink Sink `json:"sink,omitempty"` - EnvVariables map[string]string `json:"env_variables,omitempty"` - Replicas int `json:"replicas"` - SinkType string `json:"sink_type"` - Team string `json:"team"` - - FlinkName string `json:"flink_name,omitempty"` - DeploymentID string `json:"deployment_id,omitempty"` - JobId string `json:"job_id,omitempty"` - Savepoint any `json:"savepoint,omitempty"` - ChartValues *ChartValues `json:"chart_values,omitempty"` - Deleted bool `json:"deleted,omitempty"` - Namespace string `json:"namespace,omitempty"` - PrometheusURL string `json:"prometheus_url,omitempty"` - JarURI string `json:"jar_uri,omitempty"` + Resources Resources `json:"resources,omitempty"` + Source []Source `json:"source,omitempty"` + Sink Sink `json:"sink,omitempty"` + EnvVariables map[string]string `json:"env_variables,omitempty"` + Replicas int `json:"replicas"` + SinkType string `json:"sink_type"` + Team string `json:"team"` + FlinkName string `json:"flink_name,omitempty"` + DeploymentID string `json:"deployment_id,omitempty"` + Savepoint any `json:"savepoint,omitempty"` + ChartValues *ChartValues `json:"chart_values,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Namespace string `json:"namespace,omitempty"` + PrometheusURL string `json:"prometheus_url,omitempty"` + JarURI string `json:"jar_uri,omitempty"` } type ChartValues struct { @@ -228,6 +226,7 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo //transform resource name to safe length //transformation #5 + //TODO: build name from title as project--dagger cfg.EnvVariables[keyFlinkJobID] = r.Name //transformation #6 diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go index 53ba11c..349cf7d 100644 --- a/modules/dagger/driver.go +++ b/modules/dagger/driver.go @@ -57,11 +57,13 @@ type daggerDriver struct { conf driverConf kubeDeploy kubeDeployFn kubeGetPod kubeGetPodFn + kubeGetCRD kubeGetCRDFn } type ( kubeDeployFn func(ctx context.Context, isCreate bool, conf kube.Config, hc helm.ReleaseConfig) error kubeGetPodFn func(ctx context.Context, conf kube.Config, ns string, labels map[string]string) ([]kube.Pod, error) + kubeGetCRDFn func(ctx context.Context, conf kube.Config, ns string, name string) (kube.FlinkDeploymentStatus, error) ) type driverConf struct { @@ -88,9 +90,10 @@ type Output struct { State string `json:"state,omitempty"` JMDeployStatus string `json:"jm_deploy_status,omitempty"` JobStatus string `json:"job_status,omitempty"` - Namespace string `json:"namespace,omitempty"` - ReleaseName string `json:"release_name,omitempty"` + Reconcilation string `json:"reconcilation,omitempty"` Pods []kube.Pod `json:"pods,omitempty"` + Namespace string `json:"namespace,omitempty"` + JobID string `json:"job_id,omitempty"` } type transientData struct { diff --git a/modules/dagger/driver_output.go b/modules/dagger/driver_output.go index 7f2e819..6b3f387 100644 --- a/modules/dagger/driver_output.go +++ b/modules/dagger/driver_output.go @@ -39,12 +39,22 @@ func (dd *daggerDriver) refreshOutput(ctx context.Context, r resource.Resource, return nil, err } - pods, err := dd.kubeGetPod(ctx, kubeOut.Configs, rc.Namespace, map[string]string{"app": rc.Name}) + pods, err := dd.kubeGetPod(ctx, kubeOut.Configs, rc.Namespace, map[string]string{"app": conf.DeploymentID}) if err != nil { return nil, errors.ErrInternal.WithCausef(err.Error()) } output.Pods = pods output.Namespace = conf.Namespace + output.JobID = conf.DeploymentID + + crd, err := dd.kubeGetCRD(ctx, kubeOut.Configs, rc.Namespace, rc.Name) + if err != nil { + return nil, errors.ErrInternal.WithCausef(err.Error()) + } + + output.JMDeployStatus = crd.JMDeployStatus + output.JobStatus = crd.JobStatus + output.Reconcilation = crd.Reconciliation return modules.MustJSON(output), nil } diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go index 91c12f8..30c7540 100644 --- a/modules/dagger/driver_plan.go +++ b/modules/dagger/driver_plan.go @@ -47,8 +47,7 @@ func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.Actio exr.Resource.State = resource.State{ Status: resource.StatusPending, Output: modules.MustJSON(Output{ - Namespace: conf.Namespace, - ReleaseName: conf.DeploymentID, + Namespace: conf.Namespace, }), NextSyncAt: &immediately, ModuleData: modules.MustJSON(transientData{ diff --git a/modules/dagger/module.go b/modules/dagger/module.go index 7323ab6..6037d70 100644 --- a/modules/dagger/module.go +++ b/modules/dagger/module.go @@ -8,16 +8,24 @@ import ( "github.com/goto/entropy/core/module" "github.com/goto/entropy/modules/flink" + "github.com/goto/entropy/pkg/errors" "github.com/goto/entropy/pkg/helm" "github.com/goto/entropy/pkg/kube" "github.com/goto/entropy/pkg/validator" "helm.sh/helm/v3/pkg/release" + v1 "k8s.io/api/core/v1" ) const ( keyFlinkDependency = "flink" ) +type FlinkCRDStatus struct { + JobManagerDeploymentStatus string `json:"jobManagerDeploymentStatus"` + JobStatus string `json:"jobStatus"` + ReconciliationStatus string `json:"reconciliationStatus"` +} + var Module = module.Descriptor{ Kind: "dagger", Dependencies: map[string]string{ @@ -61,7 +69,54 @@ var Module = module.Descriptor{ _, errHelm := helmCl.Upsert(&hc, canUpdate) return errHelm }, - kubeGetPod: nil, - }, nil + kubeGetPod: func(ctx context.Context, conf kube.Config, ns string, labels map[string]string) ([]kube.Pod, error) { + kubeCl, err := kube.NewClient(ctx, conf) + if err != nil { + return nil, errors.ErrInternal.WithMsgf("failed to create new kube client on firehose driver kube get pod").WithCausef(err.Error()) + } + return kubeCl.GetPodDetails(ctx, ns, labels, func(pod v1.Pod) bool { + // allow pods that are in running state and are not marked for deletion + return pod.Status.Phase == v1.PodRunning && pod.DeletionTimestamp == nil + }) + }, + kubeGetCRD: func(ctx context.Context, conf kube.Config, ns string, name string) (kube.FlinkDeploymentStatus, error) { + kubeCl, err := kube.NewClient(ctx, conf) + if err != nil { + return kube.FlinkDeploymentStatus{}, errors.ErrInternal.WithMsgf("failed to create new kube client on firehose driver kube get pod").WithCausef(err.Error()) + } + crd, err := kubeCl.GetCRDDetails(ctx, ns, name) + if err != nil { + return kube.FlinkDeploymentStatus{}, err + } + flinkDeployment := crd.Object + + var flinkCRDStatus FlinkCRDStatus + statusInterface, ok := flinkDeployment["status"].(map[string]interface{}) + if !ok { + return kube.FlinkDeploymentStatus{}, errors.ErrInternal.WithMsgf("failed to convert flink deployment status to map[string]interface{}") + } + + if jmStatus, ok := statusInterface["jobManagerDeploymentStatus"].(string); ok { + flinkCRDStatus.JobManagerDeploymentStatus = jmStatus + } + if jobStatus, ok := statusInterface["jobStatus"].(map[string]interface{}); ok { + if st, ok := jobStatus["state"].(string); ok { + flinkCRDStatus.JobStatus = st + } + } + if reconciliationStatus, ok := statusInterface["reconciliationStatus"].(map[string]interface{}); ok { + if st, ok := reconciliationStatus["state"].(string); ok { + flinkCRDStatus.ReconciliationStatus = st + } + } + + status := kube.FlinkDeploymentStatus{ + JMDeployStatus: flinkCRDStatus.JobManagerDeploymentStatus, + JobStatus: flinkCRDStatus.JobStatus, + Reconciliation: flinkCRDStatus.ReconciliationStatus, + } + return status, nil + + }}, nil }, } diff --git a/pkg/kube/client.go b/pkg/kube/client.go index af9f6fa..b4c15b2 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -15,9 +15,12 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" typedbatchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" "k8s.io/client-go/rest" @@ -55,6 +58,13 @@ type Pod struct { Status string `json:"status"` } +type FlinkDeploymentStatus struct { + State string `json:"state"` + JMDeployStatus string `json:"jm_deploy_status"` + JobStatus string `json:"job_status"` + Reconciliation string `json:"reconciliation"` +} + type LogOptions struct { App string `mapstructure:"app"` Pod string `mapstructure:"pod"` @@ -323,6 +333,29 @@ func (c Client) GetPodDetails(ctx context.Context, namespace string, labelSelect return podDetails, nil } +func (c Client) GetCRDDetails(ctx context.Context, namespace string, name string) (*unstructured.Unstructured, error) { + // Initialize the dynamic client + dynamicClient, err := dynamic.NewForConfig(&c.restConfig) + if err != nil { + return nil, fmt.Errorf("failed to create dynamic client: %v", err) + } + + // Define the GVR (GroupVersionResource) for the FlinkDeployment CRD + gvr := schema.GroupVersionResource{ + Group: "flink.apache.org", + Version: "v1beta1", + Resource: "flinkdeployments", + } + + // Fetch the FlinkDeployment CRD details + flinkDeployment, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get FlinkDeployment: %v", err) + } + + return flinkDeployment, nil +} + func streamContainerLogs(ctx context.Context, ns, podName string, logCh chan<- LogChunk, clientSet *kubernetes.Clientset, podLogOpts corev1.PodLogOptions, ) error { From 8204a8586798f3eef13601c2f5be0d0c792c2416 Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Mon, 23 Sep 2024 11:12:22 +0530 Subject: [PATCH 10/19] feat: handle status --- modules/dagger/driver_sync.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/modules/dagger/driver_sync.go b/modules/dagger/driver_sync.go index 9398a14..e75fb9e 100644 --- a/modules/dagger/driver_sync.go +++ b/modules/dagger/driver_sync.go @@ -18,6 +18,11 @@ func (dd *daggerDriver) Sync(ctx context.Context, exr module.ExpandedResource) ( return nil, err } + out, err := readOutputData(exr) + if err != nil { + return nil, errors.ErrInternal.WithCausef(err.Error()) + } + conf, err := readConfig(exr, exr.Spec.Configs, dd.conf) if err != nil { return nil, errors.ErrInternal.WithCausef(err.Error()) @@ -56,6 +61,14 @@ func (dd *daggerDriver) Sync(ctx context.Context, exr module.ExpandedResource) ( return &finalState, nil } + finalOut, err := dd.refreshOutput(ctx, exr.Resource, *conf, *out, flinkOut.KubeCluster) + if err != nil { + return nil, err + } + finalState.Output = finalOut + + finalState.Status = resource.StatusCompleted + finalState.ModuleData = nil return &finalState, nil } From 346dcc86420458caabe8b4c86f7d0bb65a8f2e8b Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Mon, 23 Sep 2024 11:17:44 +0530 Subject: [PATCH 11/19] refactor: seperate contants by type --- modules/dagger/config.go | 68 ++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index 6b16a1c..75da0d1 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -14,28 +14,51 @@ import ( "github.com/goto/entropy/pkg/validator" ) -const helmReleaseNameMaxLength = 53 -const keyStreams = "STREAMS" -const keyFlinkJobID = "FLINK_JOB_ID" -const keySinkInfluxURL = "SINK_INFLUX_URL" -const keySinkInfluxPassword = "SINK_INFLUX_PASSWORD" -const keySinkInfluxDBName = "SINK_INFLUX_DB_NAME" -const keySinkInfluxUsername = "SINK_INFLUX_USERNAME" -const keySinkInfluxMeasurementName = "SINK_INFLUX_MEASUREMENT_NAME" -const keyRedisServer = "REDIS_SERVER" -const SourceKafkaConsumerConfigAutoCommitEnable = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE" -const SourceKafkaConsumerConfigAutoOffsetReset = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET" -const SourceKafkaConsumerConfigBootstrapServers = "SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS" -const SinkTypeInflux = "INFLUX" -const SinkTypeKafka = "KAFKA" -const SinkTypeBigquery = "BIGQUERY" -const keySinkKafkaBrokers = "SINK_KAFKA_BROKERS" -const keySinkKafkaStream = "SINK_KAFKA_STREAM" -const keySinkType = "SINK_TYPE" -const keySinkKafkaProtoMsg = "SINK_KAFKA_PROTO_MESSAGE" -const keySinkKafkaTopic = "SINK_KAFKA_TOPIC" -const keySinkKafkaKey = "SINK_KAFKA_PROTO_KEY" -const keySinkKafkaLingerMs = "SINK_KAFKA_LINGER_MS" +const ( + helmReleaseNameMaxLength = 53 +) + +// Stream-related constants +const ( + keyStreams = "STREAMS" + keySinkType = "SINK_TYPE" +) + +// Flink-related constants +const ( + keyFlinkJobID = "FLINK_JOB_ID" +) + +// Influx-related constants +const ( + keySinkInfluxURL = "SINK_INFLUX_URL" + keySinkInfluxPassword = "SINK_INFLUX_PASSWORD" + keySinkInfluxDBName = "SINK_INFLUX_DB_NAME" + keySinkInfluxUsername = "SINK_INFLUX_USERNAME" + keySinkInfluxMeasurementName = "SINK_INFLUX_MEASUREMENT_NAME" +) + +// Kafka-related constants +const ( + SourceKafkaConsumerConfigAutoCommitEnable = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_COMMIT_ENABLE" + SourceKafkaConsumerConfigAutoOffsetReset = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET" + SourceKafkaConsumerConfigBootstrapServers = "SOURCE_KAFKA_CONSUMER_CONFIG_BOOTSTRAP_SERVERS" + keySinkKafkaBrokers = "SINK_KAFKA_BROKERS" + keySinkKafkaStream = "SINK_KAFKA_STREAM" + keySinkKafkaProtoMsg = "SINK_KAFKA_PROTO_MESSAGE" + keySinkKafkaTopic = "SINK_KAFKA_TOPIC" + keySinkKafkaKey = "SINK_KAFKA_PROTO_KEY" + keySinkKafkaLingerMs = "SINK_KAFKA_LINGER_MS" +) + +// Sink types +const ( + SinkTypeInflux = "INFLUX" + SinkTypeKafka = "KAFKA" + SinkTypeBigquery = "BIGQUERY" +) + +// BigQuery-related constants const ( keySinkBigqueryGoogleCloudProjectID = "SINK_BIGQUERY_GOOGLE_CLOUD_PROJECT_ID" keySinkBigqueryDatasetName = "SINK_BIGQUERY_DATASET_NAME" @@ -205,7 +228,6 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo //transformation #12 cfg.EnvVariables[keyStreams] = string(mustMarshalJSON(source)) - fmt.Printf("cfg.EnvVariables[keyStreams]: %v\n", cfg.EnvVariables[keyStreams]) } //transformation #2 From 52e4d2a27359a4d35e076ed3b29a3d748477e1ca Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Mon, 23 Sep 2024 11:22:13 +0530 Subject: [PATCH 12/19] refactor: kubeGetCRD function --- modules/dagger/module.go | 57 ++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/modules/dagger/module.go b/modules/dagger/module.go index 6037d70..6a6d4cc 100644 --- a/modules/dagger/module.go +++ b/modules/dagger/module.go @@ -88,35 +88,36 @@ var Module = module.Descriptor{ if err != nil { return kube.FlinkDeploymentStatus{}, err } - flinkDeployment := crd.Object + return parseFlinkCRDStatus(crd.Object) + }}, nil + }, +} - var flinkCRDStatus FlinkCRDStatus - statusInterface, ok := flinkDeployment["status"].(map[string]interface{}) - if !ok { - return kube.FlinkDeploymentStatus{}, errors.ErrInternal.WithMsgf("failed to convert flink deployment status to map[string]interface{}") - } +func parseFlinkCRDStatus(flinkDeployment map[string]interface{}) (kube.FlinkDeploymentStatus, error) { + var flinkCRDStatus FlinkCRDStatus + statusInterface, ok := flinkDeployment["status"].(map[string]interface{}) + if !ok { + return kube.FlinkDeploymentStatus{}, errors.ErrInternal.WithMsgf("failed to convert flink deployment status to map[string]interface{}") + } - if jmStatus, ok := statusInterface["jobManagerDeploymentStatus"].(string); ok { - flinkCRDStatus.JobManagerDeploymentStatus = jmStatus - } - if jobStatus, ok := statusInterface["jobStatus"].(map[string]interface{}); ok { - if st, ok := jobStatus["state"].(string); ok { - flinkCRDStatus.JobStatus = st - } - } - if reconciliationStatus, ok := statusInterface["reconciliationStatus"].(map[string]interface{}); ok { - if st, ok := reconciliationStatus["state"].(string); ok { - flinkCRDStatus.ReconciliationStatus = st - } - } - - status := kube.FlinkDeploymentStatus{ - JMDeployStatus: flinkCRDStatus.JobManagerDeploymentStatus, - JobStatus: flinkCRDStatus.JobStatus, - Reconciliation: flinkCRDStatus.ReconciliationStatus, - } - return status, nil + if jmStatus, ok := statusInterface["jobManagerDeploymentStatus"].(string); ok { + flinkCRDStatus.JobManagerDeploymentStatus = jmStatus + } + if jobStatus, ok := statusInterface["jobStatus"].(map[string]interface{}); ok { + if st, ok := jobStatus["state"].(string); ok { + flinkCRDStatus.JobStatus = st + } + } + if reconciliationStatus, ok := statusInterface["reconciliationStatus"].(map[string]interface{}); ok { + if st, ok := reconciliationStatus["state"].(string); ok { + flinkCRDStatus.ReconciliationStatus = st + } + } - }}, nil - }, + status := kube.FlinkDeploymentStatus{ + JMDeployStatus: flinkCRDStatus.JobManagerDeploymentStatus, + JobStatus: flinkCRDStatus.JobStatus, + Reconciliation: flinkCRDStatus.ReconciliationStatus, + } + return status, nil } From bd1f53dd6c643eeee8441a31a391848f50b59226 Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Mon, 23 Sep 2024 13:50:16 +0530 Subject: [PATCH 13/19] feat: add dagger update action --- modules/dagger/config.go | 75 ++++++++++++++--------------------- modules/dagger/driver.go | 1 + modules/dagger/driver_plan.go | 52 +++++++++++++++++++++++- modules/dagger/driver_sync.go | 2 +- 4 files changed, 83 insertions(+), 47 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index 75da0d1..7697b62 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -195,41 +195,28 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo } //transformation #9 and #11 - if cfg.EnvVariables[keyStreams] == "" { - //transformation #1 - source := cfg.Source - - cfg.Source = []Source{} - for i := range source { - if source[i].SourceParquet.SourceParquetFilePaths != nil && len(source[i].SourceParquet.SourceParquetFilePaths) > 0 { - //source is parquete - //do nothing - cfg.Source = []Source{ - { - SourceParquet: SourceParquet{}, - }, - } - continue - } - if source[i].SourceKafkaConsumerConfigGroupID == "" { - source[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) - } - source[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] - source[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] - source[i].SourceKafkaConsumerConfigBootstrapServers = dc.EnvVariables[SourceKafkaConsumerConfigBootstrapServers] - - cfg.Source = append(cfg.Source, Source{ - SourceKafka: SourceKafka{ - SourceKafkaName: source[i].SourceKafkaName, - SourceKafkaConsumerConfigGroupID: source[i].SourceKafkaConsumerConfigGroupID, - }, - }) + //transformation #1 + source := cfg.Source + + for i := range source { + if source[i].SourceParquet.SourceParquetFilePaths != nil && len(source[i].SourceParquet.SourceParquetFilePaths) > 0 { + //source is parquete + //do nothing + continue } + //TODO: check how to handle increment group id on update + if source[i].SourceKafkaConsumerConfigGroupID == "" { + source[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) + } + source[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] + source[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] + source[i].SourceKafkaConsumerConfigBootstrapServers = dc.EnvVariables[SourceKafkaConsumerConfigBootstrapServers] - //transformation #12 - cfg.EnvVariables[keyStreams] = string(mustMarshalJSON(source)) } + //transformation #12 + cfg.EnvVariables[keyStreams] = string(mustMarshalJSON(source)) + //transformation #2 cfg.EnvVariables = modules.CloneAndMergeMaps(dc.EnvVariables, cfg.EnvVariables) @@ -239,10 +226,7 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo return nil, errors.ErrInternal.WithMsgf("invalid flink state").WithCausef(err.Error()) } - if cfg.Namespace == "" { - ns := flinkOut.KubeNamespace - cfg.Namespace = ns - } + cfg.Namespace = flinkOut.KubeNamespace //transformation #4 //transform resource name to safe length @@ -321,6 +305,7 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo return &cfg, nil } + func incrementGroupId(groupId string, step int) string { incrementNumberInString := func(number string) int { num, _ := strconv.Atoi(number) @@ -349,18 +334,18 @@ func mustMarshalJSON(v interface{}) []byte { return data } -func mergeResources(defaultResources, currResources Resources) Resources { - if currResources.TaskManager.CPU == "" { - currResources.TaskManager.CPU = defaultResources.TaskManager.CPU +func mergeResources(oldResources, newResources Resources) Resources { + if newResources.TaskManager.CPU == "" { + newResources.TaskManager.CPU = oldResources.TaskManager.CPU } - if currResources.TaskManager.Memory == "" { - currResources.TaskManager.Memory = defaultResources.TaskManager.Memory + if newResources.TaskManager.Memory == "" { + newResources.TaskManager.Memory = oldResources.TaskManager.Memory } - if currResources.JobManager.CPU == "" { - currResources.JobManager.CPU = defaultResources.JobManager.CPU + if newResources.JobManager.CPU == "" { + newResources.JobManager.CPU = oldResources.JobManager.CPU } - if currResources.JobManager.Memory == "" { - currResources.JobManager.Memory = defaultResources.JobManager.Memory + if newResources.JobManager.Memory == "" { + newResources.JobManager.Memory = oldResources.JobManager.Memory } - return currResources + return newResources } diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go index 349cf7d..1621dba 100644 --- a/modules/dagger/driver.go +++ b/modules/dagger/driver.go @@ -21,6 +21,7 @@ import ( const ( stepReleaseCreate = "release_create" + stepReleaseUpdate = "release_update" ) const ( diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go index 30c7540..ba9d46c 100644 --- a/modules/dagger/driver_plan.go +++ b/modules/dagger/driver_plan.go @@ -17,8 +17,9 @@ func (dd *daggerDriver) Plan(_ context.Context, exr module.ExpandedResource, act switch act.Name { case module.CreateAction: return dd.planCreate(exr, act) + default: - return nil, nil + return dd.planChange(exr, act) } } @@ -58,6 +59,55 @@ func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.Actio return &exr.Resource, nil } +func (dd *daggerDriver) planChange(exr module.ExpandedResource, act module.ActionRequest) (*resource.Resource, error) { + curConf, err := readConfig(exr, exr.Resource.Spec.Configs, dd.conf) + if err != nil { + return nil, err + } + + switch act.Name { + case module.UpdateAction: + newConf, err := readConfig(exr, act.Params, dd.conf) + if err != nil { + return nil, err + } + + chartVals, err := mergeChartValues(curConf.ChartValues, newConf.ChartValues) + if err != nil { + return nil, err + } + + // restore configs that are not user-controlled. + newConf.DeploymentID = curConf.DeploymentID + newConf.ChartValues = chartVals + newConf.JarURI = curConf.JarURI + + newConf.Resources = mergeResources(curConf.Resources, newConf.Resources) + + curConf = newConf + } + + immediately := dd.timeNow() + + exr.Resource.Spec.Configs = modules.MustJSON(curConf) + + err = dd.validateHelmReleaseConfigs(exr, *curConf) + if err != nil { + return nil, err + } + + exr.Resource.State = resource.State{ + Status: resource.StatusPending, + Output: exr.Resource.State.Output, + ModuleData: modules.MustJSON(transientData{ + PendingSteps: []string{stepReleaseUpdate}, + }), + NextSyncAt: &immediately, + } + + return &exr.Resource, nil +} + func (dd *daggerDriver) validateHelmReleaseConfigs(expandedResource module.ExpandedResource, config Config) error { var flinkOut flink.Output if err := json.Unmarshal(expandedResource.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { diff --git a/modules/dagger/driver_sync.go b/modules/dagger/driver_sync.go index e75fb9e..34885d1 100644 --- a/modules/dagger/driver_sync.go +++ b/modules/dagger/driver_sync.go @@ -43,7 +43,7 @@ func (dd *daggerDriver) Sync(ctx context.Context, exr module.ExpandedResource) ( modData.PendingSteps = modData.PendingSteps[1:] switch pendingStep { - case stepReleaseCreate: + case stepReleaseCreate, stepReleaseUpdate: isCreate := pendingStep == stepReleaseCreate if err := dd.releaseSync(ctx, exr.Resource, isCreate, *conf, flinkOut.KubeCluster); err != nil { return nil, err From 6efa1c4845902dacf2a40393ba241ff395f13854 Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Mon, 23 Sep 2024 14:13:23 +0530 Subject: [PATCH 14/19] fix: add Update action --- modules/dagger/module.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/dagger/module.go b/modules/dagger/module.go index 6a6d4cc..bfc4149 100644 --- a/modules/dagger/module.go +++ b/modules/dagger/module.go @@ -36,6 +36,10 @@ var Module = module.Descriptor{ Name: module.CreateAction, Description: "Creates a new dagger", }, + { + Name: module.UpdateAction, + Description: "Updates an existing dagger", + }, }, DriverFactory: func(confJSON json.RawMessage) (module.Driver, error) { conf := defaultDriverConf // clone the default value From c51c7bbc823c64969d5fe7a121228a503e7ca390 Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Mon, 23 Sep 2024 14:15:29 +0530 Subject: [PATCH 15/19] chore: change var name to sink_kafka_stream --- modules/flink/schema/config.json | 2 +- test/testbench/test_data/resource/flink_resource.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/flink/schema/config.json b/modules/flink/schema/config.json index e9e5bd4..a78df50 100644 --- a/modules/flink/schema/config.json +++ b/modules/flink/schema/config.json @@ -20,7 +20,7 @@ } } }, - "sink_kafka_stream_name": { + "sink_kafka_stream": { "type": "string" } } diff --git a/test/testbench/test_data/resource/flink_resource.json b/test/testbench/test_data/resource/flink_resource.json index ada2bf2..d68f0a5 100644 --- a/test/testbench/test_data/resource/flink_resource.json +++ b/test/testbench/test_data/resource/flink_resource.json @@ -13,7 +13,7 @@ "username": "influx-user" }, "kube_namespace": "flink-ns", - "sink_kafka_stream_name": "flinkstream" + "sink_kafka_stream": "flinkstream" }, "dependencies": [ { From fdfc892d7efb6c5ce591c983d1e3ce8457c623bb Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Mon, 23 Sep 2024 14:46:05 +0530 Subject: [PATCH 16/19] feat: merge consumer group ID if sink is same --- modules/dagger/config.go | 4 +--- modules/dagger/driver_plan.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index 7697b62..d389cda 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -211,11 +211,9 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo source[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] source[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] source[i].SourceKafkaConsumerConfigBootstrapServers = dc.EnvVariables[SourceKafkaConsumerConfigBootstrapServers] - } - //transformation #12 - cfg.EnvVariables[keyStreams] = string(mustMarshalJSON(source)) + cfg.Source = source //transformation #2 cfg.EnvVariables = modules.CloneAndMergeMaps(dc.EnvVariables, cfg.EnvVariables) diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go index ba9d46c..f18d245 100644 --- a/modules/dagger/driver_plan.go +++ b/modules/dagger/driver_plan.go @@ -29,6 +29,9 @@ func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.Actio return nil, err } + //transformation #12 + conf.EnvVariables[keyStreams] = string(mustMarshalJSON(conf.Source)) + chartVals, err := mergeChartValues(&dd.conf.ChartValues, conf.ChartValues) if err != nil { return nil, err @@ -72,6 +75,8 @@ func (dd *daggerDriver) planChange(exr module.ExpandedResource, act module.Actio return nil, err } + newConf.Source = mergeConsumerGroupId(curConf.Source, newConf.Source) + chartVals, err := mergeChartValues(curConf.ChartValues, newConf.ChartValues) if err != nil { return nil, err @@ -117,3 +122,23 @@ func (dd *daggerDriver) validateHelmReleaseConfigs(expandedResource module.Expan _, err := dd.getHelmRelease(expandedResource.Resource, config, flinkOut.KubeCluster) return err } + +func mergeConsumerGroupId(currStreams, newStreams []Source) []Source { + if len(currStreams) != len(newStreams) { + return newStreams + } + + for i := range currStreams { + if currStreams[i].SourceParquet.SourceParquetFilePaths != nil && len(currStreams[i].SourceParquet.SourceParquetFilePaths) > 0 { + //source is parquete + //do nothing + continue + } + + if currStreams[i].SourceKafka.SourceKafkaName == newStreams[i].SourceKafka.SourceKafkaName { + newStreams[i].SourceKafka.SourceKafkaConsumerConfigGroupID = currStreams[i].SourceKafka.SourceKafkaConsumerConfigGroupID + } + } + + return newStreams +} From 328f99cda7224796f6387c2f164efc220ed96ef8 Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Mon, 23 Sep 2024 15:56:27 +0530 Subject: [PATCH 17/19] feat: add start & update action --- modules/dagger/config.go | 2 ++ modules/dagger/driver.go | 3 +-- modules/dagger/driver_plan.go | 20 +++++++++++++++++++- modules/dagger/module.go | 10 ++++++++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index d389cda..fcfea25 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -111,6 +111,8 @@ type Config struct { Namespace string `json:"namespace,omitempty"` PrometheusURL string `json:"prometheus_url,omitempty"` JarURI string `json:"jar_uri,omitempty"` + State string `json:"state"` + JobState string `json:"job_state"` } type ChartValues struct { diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go index 1621dba..7142192 100644 --- a/modules/dagger/driver.go +++ b/modules/dagger/driver.go @@ -88,7 +88,6 @@ type driverConf struct { } type Output struct { - State string `json:"state,omitempty"` JMDeployStatus string `json:"jm_deploy_status,omitempty"` JobStatus string `json:"job_status,omitempty"` Reconcilation string `json:"reconcilation,omitempty"` @@ -208,7 +207,7 @@ func (dd *daggerDriver) getHelmRelease(res resource.Resource, conf Config, }, "jarURI": conf.JarURI, "programArgs": append([]string{"--encodedArgs"}, encodedProgramArgs), - "state": "running", + "state": conf.JobState, "namespace": conf.Namespace, } diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go index f18d245..4a537ff 100644 --- a/modules/dagger/driver_plan.go +++ b/modules/dagger/driver_plan.go @@ -12,6 +12,12 @@ import ( ) const SourceKafkaConsumerAutoOffsetReset = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET" +const ( + JobStateRunning = "running" + JobStateSuspended = "suspended" + StateDeployed = "DEPLOYED" + StateUserStopped = "USER_STOPPED" +) func (dd *daggerDriver) Plan(_ context.Context, exr module.ExpandedResource, act module.ActionRequest) (*resource.Resource, error) { switch act.Name { @@ -40,6 +46,8 @@ func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.Actio immediately := dd.timeNow() conf.JarURI = dd.conf.JarURI + conf.State = StateDeployed + conf.JobState = JobStateRunning exr.Resource.Spec.Configs = modules.MustJSON(conf) @@ -86,12 +94,22 @@ func (dd *daggerDriver) planChange(exr module.ExpandedResource, act module.Actio newConf.DeploymentID = curConf.DeploymentID newConf.ChartValues = chartVals newConf.JarURI = curConf.JarURI + newConf.State = StateDeployed + newConf.JobState = JobStateRunning newConf.Resources = mergeResources(curConf.Resources, newConf.Resources) curConf = newConf - } + case StopAction: + curConf.State = StateUserStopped + curConf.JobState = JobStateSuspended + + case StartAction: + curConf.State = StateDeployed + curConf.JobState = JobStateRunning + + } immediately := dd.timeNow() exr.Resource.Spec.Configs = modules.MustJSON(curConf) diff --git a/modules/dagger/module.go b/modules/dagger/module.go index bfc4149..f920672 100644 --- a/modules/dagger/module.go +++ b/modules/dagger/module.go @@ -18,6 +18,8 @@ import ( const ( keyFlinkDependency = "flink" + StopAction = "stop" + StartAction = "start" ) type FlinkCRDStatus struct { @@ -40,6 +42,14 @@ var Module = module.Descriptor{ Name: module.UpdateAction, Description: "Updates an existing dagger", }, + { + Name: StopAction, + Description: "Suspends a running dagger", + }, + { + Name: StartAction, + Description: "Starts a suspended dagger", + }, }, DriverFactory: func(confJSON json.RawMessage) (module.Driver, error) { conf := defaultDriverConf // clone the default value From cc07914623a34a662ac853f64e8c0f367ae76521 Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Tue, 24 Sep 2024 14:34:26 +0530 Subject: [PATCH 18/19] feat: add reset action --- modules/dagger/config.go | 22 ++++++++++----------- modules/dagger/driver.go | 23 +++++++++++++--------- modules/dagger/driver_plan.go | 37 +++++++++++++++++++++++++++++++++++ modules/dagger/driver_sync.go | 2 +- modules/dagger/module.go | 21 +++++++++++++++++++- 5 files changed, 82 insertions(+), 23 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index fcfea25..69eeddf 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -113,6 +113,7 @@ type Config struct { JarURI string `json:"jar_uri,omitempty"` State string `json:"state"` JobState string `json:"job_state"` + ResetOffset string `json:"reset_offset"` } type ChartValues struct { @@ -200,19 +201,16 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo //transformation #1 source := cfg.Source - for i := range source { - if source[i].SourceParquet.SourceParquetFilePaths != nil && len(source[i].SourceParquet.SourceParquetFilePaths) > 0 { - //source is parquete - //do nothing - continue + if !(source[0].SourceParquet.SourceParquetFilePaths != nil && len(source[0].SourceParquet.SourceParquetFilePaths) > 0) { + for i := range source { + //TODO: check how to handle increment group id on update + if source[i].SourceKafkaConsumerConfigGroupID == "" { + source[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) + } + source[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] + source[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] + source[i].SourceKafkaConsumerConfigBootstrapServers = dc.EnvVariables[SourceKafkaConsumerConfigBootstrapServers] } - //TODO: check how to handle increment group id on update - if source[i].SourceKafkaConsumerConfigGroupID == "" { - source[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(r.Name+"-0001", i) - } - source[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] - source[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] - source[i].SourceKafkaConsumerConfigBootstrapServers = dc.EnvVariables[SourceKafkaConsumerConfigBootstrapServers] } cfg.Source = source diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go index 7142192..2efb06d 100644 --- a/modules/dagger/driver.go +++ b/modules/dagger/driver.go @@ -22,6 +22,8 @@ import ( const ( stepReleaseCreate = "release_create" stepReleaseUpdate = "release_update" + stepReleaseStop = "release_stop" + stepKafkaReset = "kafka_reset" ) const ( @@ -54,17 +56,19 @@ var defaultDriverConf = driverConf{ } type daggerDriver struct { - timeNow func() time.Time - conf driverConf - kubeDeploy kubeDeployFn - kubeGetPod kubeGetPodFn - kubeGetCRD kubeGetCRDFn + timeNow func() time.Time + conf driverConf + kubeDeploy kubeDeployFn + kubeGetPod kubeGetPodFn + kubeGetCRD kubeGetCRDFn + consumerReset consumerResetFn } type ( - kubeDeployFn func(ctx context.Context, isCreate bool, conf kube.Config, hc helm.ReleaseConfig) error - kubeGetPodFn func(ctx context.Context, conf kube.Config, ns string, labels map[string]string) ([]kube.Pod, error) - kubeGetCRDFn func(ctx context.Context, conf kube.Config, ns string, name string) (kube.FlinkDeploymentStatus, error) + kubeDeployFn func(ctx context.Context, isCreate bool, conf kube.Config, hc helm.ReleaseConfig) error + kubeGetPodFn func(ctx context.Context, conf kube.Config, ns string, labels map[string]string) ([]kube.Pod, error) + kubeGetCRDFn func(ctx context.Context, conf kube.Config, ns string, name string) (kube.FlinkDeploymentStatus, error) + consumerResetFn func(ctx context.Context, conf Config, resetTo string) []Source ) type driverConf struct { @@ -97,7 +101,8 @@ type Output struct { } type transientData struct { - PendingSteps []string `json:"pending_steps"` + PendingSteps []string `json:"pending_steps"` + ResetOffsetTo string `json:"reset_offset_to,omitempty"` } func mergeChartValues(cur, newVal *ChartValues) (*ChartValues, error) { diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go index 4a537ff..9fd5a96 100644 --- a/modules/dagger/driver_plan.go +++ b/modules/dagger/driver_plan.go @@ -9,6 +9,7 @@ import ( "github.com/goto/entropy/modules" "github.com/goto/entropy/modules/flink" "github.com/goto/entropy/pkg/errors" + "github.com/goto/entropy/pkg/kafka" ) const SourceKafkaConsumerAutoOffsetReset = "SOURCE_KAFKA_CONSUMER_CONFIG_AUTO_OFFSET_RESET" @@ -24,6 +25,9 @@ func (dd *daggerDriver) Plan(_ context.Context, exr module.ExpandedResource, act case module.CreateAction: return dd.planCreate(exr, act) + case ResetAction: + return dd.planReset(exr, act) + default: return dd.planChange(exr, act) } @@ -131,6 +135,39 @@ func (dd *daggerDriver) planChange(exr module.ExpandedResource, act module.Actio return &exr.Resource, nil } +func (dd *daggerDriver) planReset(exr module.ExpandedResource, act module.ActionRequest) (*resource.Resource, error) { + resetValue, err := kafka.ParseResetV2Params(act.Params) + if err != nil { + return nil, err + } + + immediately := dd.timeNow() + + curConf, err := readConfig(exr, exr.Resource.Spec.Configs, dd.conf) + if err != nil { + return nil, err + } + + curConf.ResetOffset = resetValue + + curConf.Source = dd.consumerReset(context.Background(), *curConf, resetValue) + curConf.EnvVariables[keyStreams] = string(mustMarshalJSON(curConf.Source)) + + exr.Resource.Spec.Configs = modules.MustJSON(curConf) + exr.Resource.State = resource.State{ + Status: resource.StatusPending, + Output: exr.Resource.State.Output, + NextSyncAt: &immediately, + ModuleData: modules.MustJSON(transientData{ + ResetOffsetTo: resetValue, + PendingSteps: []string{ + stepKafkaReset, + }, + }), + } + return &exr.Resource, nil +} + func (dd *daggerDriver) validateHelmReleaseConfigs(expandedResource module.ExpandedResource, config Config) error { var flinkOut flink.Output if err := json.Unmarshal(expandedResource.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { diff --git a/modules/dagger/driver_sync.go b/modules/dagger/driver_sync.go index 34885d1..f5ebdcd 100644 --- a/modules/dagger/driver_sync.go +++ b/modules/dagger/driver_sync.go @@ -43,7 +43,7 @@ func (dd *daggerDriver) Sync(ctx context.Context, exr module.ExpandedResource) ( modData.PendingSteps = modData.PendingSteps[1:] switch pendingStep { - case stepReleaseCreate, stepReleaseUpdate: + case stepReleaseCreate, stepReleaseUpdate, stepReleaseStop, stepKafkaReset: isCreate := pendingStep == stepReleaseCreate if err := dd.releaseSync(ctx, exr.Resource, isCreate, *conf, flinkOut.KubeCluster); err != nil { return nil, err diff --git a/modules/dagger/module.go b/modules/dagger/module.go index f920672..47da1f8 100644 --- a/modules/dagger/module.go +++ b/modules/dagger/module.go @@ -20,6 +20,7 @@ const ( keyFlinkDependency = "flink" StopAction = "stop" StartAction = "start" + ResetAction = "reset" ) type FlinkCRDStatus struct { @@ -50,6 +51,10 @@ var Module = module.Descriptor{ Name: StartAction, Description: "Starts a suspended dagger", }, + { + Name: ResetAction, + Description: "Resets the offset of a dagger", + }, }, DriverFactory: func(confJSON json.RawMessage) (module.Driver, error) { conf := defaultDriverConf // clone the default value @@ -103,7 +108,9 @@ var Module = module.Descriptor{ return kube.FlinkDeploymentStatus{}, err } return parseFlinkCRDStatus(crd.Object) - }}, nil + }, + consumerReset: consumerReset, + }, nil }, } @@ -135,3 +142,15 @@ func parseFlinkCRDStatus(flinkDeployment map[string]interface{}) (kube.FlinkDepl } return status, nil } + +func consumerReset(ctx context.Context, conf Config, resetTo string) []Source { + baseGroup := conf.Source[0].SourceKafkaConsumerConfigGroupID + groupId := incrementGroupId(baseGroup, len(conf.Source)) + + for i := range conf.Source { + conf.Source[i].SourceKafkaConsumerConfigGroupID = incrementGroupId(groupId, i) + conf.Source[i].SourceKafkaConsumerConfigAutoOffsetReset = resetTo + } + + return conf.Source +} From 9cbd08127485af632d1ec9f286f4757ffd0eb8ea Mon Sep 17 00:00:00 2001 From: Ishan Arya <ishan.arya@gojek.com> Date: Wed, 25 Sep 2024 17:07:00 +0530 Subject: [PATCH 19/19] feat: create savepoint --- modules/dagger/config.go | 38 +++++++++++++++++------------------ modules/dagger/driver.go | 18 +++++++++-------- modules/dagger/driver_plan.go | 33 ++++++++++++++++++++++++++++++ modules/dagger/driver_sync.go | 2 +- modules/dagger/module.go | 17 ++++++++++++---- 5 files changed, 76 insertions(+), 32 deletions(-) diff --git a/modules/dagger/config.go b/modules/dagger/config.go index 69eeddf..86881b6 100644 --- a/modules/dagger/config.go +++ b/modules/dagger/config.go @@ -96,24 +96,25 @@ type Resources struct { } type Config struct { - Resources Resources `json:"resources,omitempty"` - Source []Source `json:"source,omitempty"` - Sink Sink `json:"sink,omitempty"` - EnvVariables map[string]string `json:"env_variables,omitempty"` - Replicas int `json:"replicas"` - SinkType string `json:"sink_type"` - Team string `json:"team"` - FlinkName string `json:"flink_name,omitempty"` - DeploymentID string `json:"deployment_id,omitempty"` - Savepoint any `json:"savepoint,omitempty"` - ChartValues *ChartValues `json:"chart_values,omitempty"` - Deleted bool `json:"deleted,omitempty"` - Namespace string `json:"namespace,omitempty"` - PrometheusURL string `json:"prometheus_url,omitempty"` - JarURI string `json:"jar_uri,omitempty"` - State string `json:"state"` - JobState string `json:"job_state"` - ResetOffset string `json:"reset_offset"` + Resources Resources `json:"resources,omitempty"` + Source []Source `json:"source,omitempty"` + Sink Sink `json:"sink,omitempty"` + EnvVariables map[string]string `json:"env_variables,omitempty"` + Replicas int `json:"replicas" default:"1"` + SinkType string `json:"sink_type"` + Team string `json:"team"` + FlinkName string `json:"flink_name,omitempty"` + DeploymentID string `json:"deployment_id,omitempty"` + Savepoint any `json:"savepoint,omitempty"` + ChartValues *ChartValues `json:"chart_values,omitempty"` + Deleted bool `json:"deleted,omitempty"` + Namespace string `json:"namespace,omitempty"` + PrometheusURL string `json:"prometheus_url,omitempty"` + JarURI string `json:"jar_uri,omitempty"` + State string `json:"state"` + JobState string `json:"job_state"` + ResetOffset string `json:"reset_offset"` + SavepointTriggerNonce int `json:"savepoint_trigger_nonce,omitempty"` } type ChartValues struct { @@ -209,7 +210,6 @@ func readConfig(r module.ExpandedResource, confJSON json.RawMessage, dc driverCo } source[i].SourceKafkaConsumerConfigAutoCommitEnable = dc.EnvVariables[SourceKafkaConsumerConfigAutoCommitEnable] source[i].SourceKafkaConsumerConfigAutoOffsetReset = dc.EnvVariables[SourceKafkaConsumerConfigAutoOffsetReset] - source[i].SourceKafkaConsumerConfigBootstrapServers = dc.EnvVariables[SourceKafkaConsumerConfigBootstrapServers] } } diff --git a/modules/dagger/driver.go b/modules/dagger/driver.go index 2efb06d..abc8f21 100644 --- a/modules/dagger/driver.go +++ b/modules/dagger/driver.go @@ -20,10 +20,11 @@ import ( ) const ( - stepReleaseCreate = "release_create" - stepReleaseUpdate = "release_update" - stepReleaseStop = "release_stop" - stepKafkaReset = "kafka_reset" + stepReleaseCreate = "release_create" + stepReleaseUpdate = "release_update" + stepReleaseStop = "release_stop" + stepKafkaReset = "kafka_reset" + stepSavepointCreate = "savepoint_create" ) const ( @@ -210,10 +211,11 @@ func (dd *daggerDriver) getHelmRelease(res resource.Resource, conf Config, "memory": conf.Resources.JobManager.Memory, }, }, - "jarURI": conf.JarURI, - "programArgs": append([]string{"--encodedArgs"}, encodedProgramArgs), - "state": conf.JobState, - "namespace": conf.Namespace, + "jarURI": conf.JarURI, + "programArgs": append([]string{"--encodedArgs"}, encodedProgramArgs), + "state": conf.JobState, + "namespace": conf.Namespace, + "savepointTriggerNonce": conf.SavepointTriggerNonce, } return rc, nil diff --git a/modules/dagger/driver_plan.go b/modules/dagger/driver_plan.go index 9fd5a96..ba7f190 100644 --- a/modules/dagger/driver_plan.go +++ b/modules/dagger/driver_plan.go @@ -28,6 +28,9 @@ func (dd *daggerDriver) Plan(_ context.Context, exr module.ExpandedResource, act case ResetAction: return dd.planReset(exr, act) + case TriggerSavepointAction: + return dd.planTriggerSavepoint(exr) + default: return dd.planChange(exr, act) } @@ -52,6 +55,7 @@ func (dd *daggerDriver) planCreate(exr module.ExpandedResource, act module.Actio conf.JarURI = dd.conf.JarURI conf.State = StateDeployed conf.JobState = JobStateRunning + conf.SavepointTriggerNonce = 1 exr.Resource.Spec.Configs = modules.MustJSON(conf) @@ -168,6 +172,35 @@ func (dd *daggerDriver) planReset(exr module.ExpandedResource, act module.Action return &exr.Resource, nil } +func (dd *daggerDriver) planTriggerSavepoint(exr module.ExpandedResource) (*resource.Resource, error) { + curConf, err := readConfig(exr, exr.Resource.Spec.Configs, dd.conf) + if err != nil { + return nil, err + } + + curConf.SavepointTriggerNonce += 1 + + immediately := dd.timeNow() + + exr.Resource.Spec.Configs = modules.MustJSON(curConf) + + err = dd.validateHelmReleaseConfigs(exr, *curConf) + if err != nil { + return nil, err + } + + exr.Resource.State = resource.State{ + Status: resource.StatusPending, + Output: exr.Resource.State.Output, + ModuleData: modules.MustJSON(transientData{ + PendingSteps: []string{stepSavepointCreate}, + }), + NextSyncAt: &immediately, + } + + return &exr.Resource, nil +} + func (dd *daggerDriver) validateHelmReleaseConfigs(expandedResource module.ExpandedResource, config Config) error { var flinkOut flink.Output if err := json.Unmarshal(expandedResource.Dependencies[keyFlinkDependency].Output, &flinkOut); err != nil { diff --git a/modules/dagger/driver_sync.go b/modules/dagger/driver_sync.go index f5ebdcd..f317c82 100644 --- a/modules/dagger/driver_sync.go +++ b/modules/dagger/driver_sync.go @@ -43,7 +43,7 @@ func (dd *daggerDriver) Sync(ctx context.Context, exr module.ExpandedResource) ( modData.PendingSteps = modData.PendingSteps[1:] switch pendingStep { - case stepReleaseCreate, stepReleaseUpdate, stepReleaseStop, stepKafkaReset: + case stepReleaseCreate, stepReleaseUpdate, stepReleaseStop, stepKafkaReset, stepSavepointCreate: isCreate := pendingStep == stepReleaseCreate if err := dd.releaseSync(ctx, exr.Resource, isCreate, *conf, flinkOut.KubeCluster); err != nil { return nil, err diff --git a/modules/dagger/module.go b/modules/dagger/module.go index 47da1f8..1404be2 100644 --- a/modules/dagger/module.go +++ b/modules/dagger/module.go @@ -17,10 +17,11 @@ import ( ) const ( - keyFlinkDependency = "flink" - StopAction = "stop" - StartAction = "start" - ResetAction = "reset" + keyFlinkDependency = "flink" + StopAction = "stop" + StartAction = "start" + ResetAction = "reset" + TriggerSavepointAction = "savepoint" ) type FlinkCRDStatus struct { @@ -55,6 +56,14 @@ var Module = module.Descriptor{ Name: ResetAction, Description: "Resets the offset of a dagger", }, + { + Name: ResetAction, + Description: "Resets the offset of a dagger", + }, + { + Name: TriggerSavepointAction, + Description: "Trigger a savepoint for a dagger", + }, }, DriverFactory: func(confJSON json.RawMessage) (module.Driver, error) { conf := defaultDriverConf // clone the default value