diff --git a/compose/demo-distributed/compose.yaml b/compose/demo-distributed/compose.yaml index aec4d6a..dce9ddf 100644 --- a/compose/demo-distributed/compose.yaml +++ b/compose/demo-distributed/compose.yaml @@ -9,13 +9,14 @@ services: - "8080:8080" environment: WOMBAT_MODE: aggregator - QUARKUS_DATASOURCE_JDBC_URL: "jdbc:sqlite:/var/data/local.db" + PERSISTENCE_BACKEND_MODE: sqlite + PERSISTENCE_BACKEND_SQLITE_JDBC_URL: "jdbc:sqlite:/var/data/local.db" # Fold sampling rows every 5 minutes instead of every hour, so pushed activity reaches the UI within minutes - COMPACTION_CRON: "0 1/5 * * * ?" - COMPACTION_WINDOW: 5m - COMPACTION_DELAY: 30s - CONNECTOR_BOAVIZTA_URI: "http://boavizta:5000/v1/" - CONNECTOR_ECOLOGITS_URI: "http://ecologits:80/" + PERSISTENCE_COMPACTION_CRON: "0 1/5 * * * ?" + PERSISTENCE_COMPACTION_WINDOW: 5m + PERSISTENCE_COMPACTION_DELAY: 30s + WOMBAT_CONNECTOR_BOAVIZTA_URI: "http://boavizta:5000/v1/" + WOMBAT_CONNECTOR_ECOLOGITS_URI: "http://ecologits:80/" # Secret of the Anthropic asset of aggregator/monitored-environments.yaml, which is evaluated here ANTHROPIC_ADMIN_KEY: "${ANTHROPIC_ADMIN_KEY:-}" volumes: diff --git a/compose/demo-standalone/application.yaml.dist b/compose/demo-standalone/application.yaml.dist index 320a93a..7e1596d 100644 --- a/compose/demo-standalone/application.yaml.dist +++ b/compose/demo-standalone/application.yaml.dist @@ -1,37 +1,34 @@ -quarkus: - datasource: - jdbc: - url: "jdbc:sqlite:/var/data/local.db" +persistence: + backend: + mode: sqlite + #mode: postgresql + sqlite: + jdbc-url: "jdbc:sqlite:/var/data/local.db" +# # Local S3 Backup setup (e.g. SeaweedFS service) +# backup: +# enabled: false +# cron: "0 0 * * * ?" # hourly +# s3: +# key-prefix: sqlite/ +# endpoint: http://localhost:8333 +# bucket: wombat-bucket +# region: us-east-1 +# access-key: admin +# secret-key: secret +# # Alternative PostgreSQL backend: +# postgresql: +# jdbc-url: "jdbc:postgresql://localhost:5432/wombat" +# username: wombat +# password: wombat -# Alternative PostgreSQL Datasource for dev: -# quarkus: -# datasource: -# db-kind: postgresql -# username: wombat -# password: wombat -# jdbc: -# url: "jdbc:postgresql://localhost:5432/wombat" - -connector: - boavizta: - uri: "http://boavizta:5000/v1/" - ecologits: - uri: "http://ecologits:80/" - -# Local S3 Backup setup (e.g. SeaweedFS service) -# backup: -# enabled: false -# cron: "0 0 * * * ?" # hourly -# s3: -# key-prefix: sqlite/ -# endpoint: http://localhost:8333 -# bucket: wombat-bucket -# region: us-east-1 -# access-key: admin -# secret-key: secret - -# Optional Secret resolution directory: -# wombat: +wombat: +# # Optional Secret resolution directory: # secret: # directory: # path: /secrets + + connector: + boavizta: + uri: "http://boavizta:5000/v1/" + ecologits: + uri: "http://ecologits:80/" diff --git a/doc/compaction.md b/doc/compaction.md index 2958b55..abf186a 100644 --- a/doc/compaction.md +++ b/doc/compaction.md @@ -4,11 +4,11 @@ Sourced metrics are written as raw samples every 5 minutes, then periodically fo The dashboard and the HTTP API only read compacted rows, so the compaction window is the delay before fresh activity becomes visible. -| Property | Default | Description | -|----------------------|---------------|--------------------------------------------------------------------------------------------------| -| `compaction.cron` | `0 1 * * * ?` | When compaction runs. Only closed windows are folded, so running it more often than they close is wasted. | -| `compaction.window` | `1h` | Width of the compacted windows. Must be a multiple of the 5-minute sampling step and divide an hour (`5m`, `10m`, `15m`, `20m`, `30m`, `1h`); the app refuses to start otherwise. | -| `compaction.delay` | `1m` | How long after its end a window is left alone, so the last raw samples land in it before it is folded. | +| Property | Default | Description | +|---------------------------------|---------------|--------------------------------------------------------------------------------------------------| +| `persistence.compaction.cron` | `0 1 * * * ?` | When compaction runs. Only closed windows are folded, so running it more often than they close is wasted. | +| `persistence.compaction.window` | `1h` | Width of the compacted windows. Must be a multiple of the 5-minute sampling step and divide an hour (`5m`, `10m`, `15m`, `20m`, `30m`, `1h`); the app refuses to start otherwise. | +| `persistence.compaction.delay` | `1m` | How long after its end a window is left alone, so the last raw samples land in it before it is folded. | > [!IMPORTANT] > Keeping the default settings in production is highly recommended. A shorter window is meant for development, where diff --git a/doc/distributed.md b/doc/distributed.md index 4850458..c8a1522 100644 --- a/doc/distributed.md +++ b/doc/distributed.md @@ -99,9 +99,7 @@ a minute. A declaration that fails is logged, and the next one tries again. Remo removes it from the aggregator at the worker's next startup. No metric is written to the worker's database, so a worker always runs on an in-memory SQLite database, whatever its -`quarkus.datasource.db-kind` and `quarkus.datasource.jdbc.url`, and logs a warning when the configured kind is another -one: it needs no database file, and loses its users and API keys at each restart. The kind is fixed at build time, so -run workers from a SQLite build. Its compaction runs on empty tables, and its dashboard stays empty. +`persistence.backend.mode` setting. Each window is sent in at most one request per metric family. A push that fails (aggregator unreachable, rejected key) is retried a few times on network errors, then logged and dropped: the worker does not buffer windows it could not send. diff --git a/doc/docker/README.md b/doc/docker/README.md index f8786e7..33fbedf 100644 --- a/doc/docker/README.md +++ b/doc/docker/README.md @@ -52,16 +52,17 @@ services: With the following `application.yaml` file: ```yaml -quarkus: - datasource: - jdbc: - url: "jdbc:sqlite:/var/data/local.db" - -connector: - boavizta: - uri: "http://boavizta:5000/v1/" - ecologits: - uri: "http://ecologits:80/" +persistence: + backend: + sqlite: + jdbc-url: "jdbc:sqlite:/var/data/local.db" + +wombat: + connector: + boavizta: + uri: "http://boavizta:5000/v1/" + ecologits: + uri: "http://ecologits:80/" ``` And the following `monitored-environments.yaml` file: diff --git a/src/main/java/tech/illuin/wombat/commons/NodeMode.java b/src/main/java/tech/illuin/wombat/commons/NodeMode.java index a9b1de3..210ad72 100644 --- a/src/main/java/tech/illuin/wombat/commons/NodeMode.java +++ b/src/main/java/tech/illuin/wombat/commons/NodeMode.java @@ -3,9 +3,9 @@ public enum NodeMode { /** Samples its environments into its own database and serves them: the single-instance setup. */ - STANDALONE, + standalone, /** Stores and serves the metrics pushed by workers on {@code POST /api/workers/{worker}/metrics}, and samples nothing itself. */ - AGGREGATOR, + aggregator, /** Samples its environments and pushes the metrics to an aggregator instead of its own database. */ - WORKER, + worker, } diff --git a/src/main/java/tech/illuin/wombat/commons/NodeModeConfigSourceFactory.java b/src/main/java/tech/illuin/wombat/commons/NodeModeConfigSourceFactory.java index 6e1da75..0882df1 100644 --- a/src/main/java/tech/illuin/wombat/commons/NodeModeConfigSourceFactory.java +++ b/src/main/java/tech/illuin/wombat/commons/NodeModeConfigSourceFactory.java @@ -11,13 +11,14 @@ import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; +import static tech.illuin.wombat.persistence.PersistenceBackendConfigSourceFactory.ACTIVE_BACKEND_CONFIG_KEY; + @StaticInitSafe public class NodeModeConfigSourceFactory implements ConfigSourceFactory { - public static final String AGGREGATOR = "_internal.aggregator"; + public static final String AGGREGATOR_CONFIG_KEY = "_internal.aggregator"; private static final String WORKER_DB_KIND = "sqlite"; private static final String WORKER_JDBC_URL = "jdbc:sqlite::memory:"; @@ -30,18 +31,19 @@ public Iterable getConfigSources(ConfigSourceContext context) NodeMode mode = value.getValue() == null ? null : parse(value.getValue()); Map properties = new HashMap<>(); - properties.put(AGGREGATOR, String.valueOf(mode == NodeMode.AGGREGATOR)); + properties.put(AGGREGATOR_CONFIG_KEY, String.valueOf(mode == NodeMode.aggregator)); - if (mode == NodeMode.WORKER) + if (mode == NodeMode.worker) { - String dbKind = context.getValue("quarkus.datasource.db-kind").getValue(); - if (dbKind != null && !dbKind.equals(WORKER_DB_KIND)) - logger.warn("Worker mode only runs on SQLite: overriding quarkus.datasource.db-kind {} with {}", dbKind, WORKER_DB_KIND); + String backend = context.getValue(ACTIVE_BACKEND_CONFIG_KEY).getValue(); + if (backend != null && !backend.equals(WORKER_DB_KIND)) + logger.warn("Worker mode only runs on SQLite in-memory: overriding backend {} with {}", backend, WORKER_DB_KIND); properties.put("quarkus.datasource.db-kind", WORKER_DB_KIND); properties.put("quarkus.datasource.jdbc.url", WORKER_JDBC_URL); properties.put("quarkus.datasource.jdbc.min-size", "1"); properties.put("quarkus.datasource.jdbc.max-size", "1"); + properties.put(ACTIVE_BACKEND_CONFIG_KEY, WORKER_DB_KIND); } return List.of(new PropertiesConfigSource( @@ -53,6 +55,6 @@ public Iterable getConfigSources(ConfigSourceContext context) private static NodeMode parse(String mode) { - return NodeMode.valueOf(mode.trim().replace('-', '_').toUpperCase(Locale.ROOT)); + return NodeMode.valueOf(mode.trim().replace('-', '_').toLowerCase()); } } diff --git a/src/main/java/tech/illuin/wombat/commons/WombatProperties.java b/src/main/java/tech/illuin/wombat/commons/WombatProperties.java index d90ed2e..abc2968 100644 --- a/src/main/java/tech/illuin/wombat/commons/WombatProperties.java +++ b/src/main/java/tech/illuin/wombat/commons/WombatProperties.java @@ -2,6 +2,11 @@ import io.smallrye.config.ConfigMapping; import io.smallrye.config.WithDefault; +import tech.illuin.wombat.engine.connector.boavizta.BoaviztaProperties; +import tech.illuin.wombat.engine.connector.ecologits.EcologitsProperties; +import tech.illuin.wombat.engine.evaluation.CachedAssetEvaluatorProperties; +import tech.illuin.wombat.engine.module.extension.WombatModuleProperties; +import tech.illuin.wombat.engine.monitor.MonitorProperties; import tech.illuin.wombat.persistence.backend.push.WorkerProperties; @ConfigMapping(prefix = "wombat") @@ -11,4 +16,24 @@ public interface WombatProperties NodeMode mode(); WorkerProperties worker(); + + MonitorProperties monitor(); + + EvaluatorProperties evaluator(); + + WombatModuleProperties module(); + + ConnectorProperties connector(); + + interface EvaluatorProperties + { + CachedAssetEvaluatorProperties cache(); + } + + interface ConnectorProperties + { + BoaviztaProperties boavizta(); + + EcologitsProperties ecologits(); + } } diff --git a/src/main/java/tech/illuin/wombat/endpoint/AppController.java b/src/main/java/tech/illuin/wombat/endpoint/AppController.java index c7f9574..034bbff 100644 --- a/src/main/java/tech/illuin/wombat/endpoint/AppController.java +++ b/src/main/java/tech/illuin/wombat/endpoint/AppController.java @@ -9,9 +9,11 @@ import org.eclipse.microprofile.openapi.annotations.media.Schema; import org.eclipse.microprofile.openapi.annotations.media.SchemaProperty; import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import tech.illuin.wombat.commons.WombatProperties; import tech.illuin.wombat.endpoint.response.Response; import tech.illuin.wombat.endpoint.response.Status; import tech.illuin.wombat.security.AuthProperties; +import tech.illuin.wombat.endpoint.UIProperties; import java.util.Locale; diff --git a/src/main/java/tech/illuin/wombat/endpoint/MetricController.java b/src/main/java/tech/illuin/wombat/endpoint/MetricController.java index 82b44c3..2d3def3 100644 --- a/src/main/java/tech/illuin/wombat/endpoint/MetricController.java +++ b/src/main/java/tech/illuin/wombat/endpoint/MetricController.java @@ -14,7 +14,6 @@ import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import tech.illuin.wombat.commons.NodeMode; import tech.illuin.wombat.commons.NodeModeConfigSourceFactory; import tech.illuin.wombat.core.asset.AssetIdentity; import tech.illuin.wombat.core.source.data.MetricData; @@ -42,7 +41,7 @@ * are saved. */ @Path("api/workers/{worker}/metrics") -@EndpointDisabled(name = NodeModeConfigSourceFactory.AGGREGATOR, stringValue = "false") +@EndpointDisabled(name = NodeModeConfigSourceFactory.AGGREGATOR_CONFIG_KEY, stringValue = "false") @PermissionsAllowed(Permission.Names.API_METRIC_WRITE) public class MetricController { diff --git a/src/main/java/tech/illuin/wombat/endpoint/WorkerController.java b/src/main/java/tech/illuin/wombat/endpoint/WorkerController.java index 0cd2dae..e920e88 100644 --- a/src/main/java/tech/illuin/wombat/endpoint/WorkerController.java +++ b/src/main/java/tech/illuin/wombat/endpoint/WorkerController.java @@ -16,7 +16,6 @@ import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import tech.illuin.wombat.commons.NodeMode; import tech.illuin.wombat.commons.NodeModeConfigSourceFactory; import tech.illuin.wombat.core.asset.Asset; import tech.illuin.wombat.core.asset.Environment; @@ -34,7 +33,6 @@ import java.util.LinkedHashMap; import java.util.Map; -import java.util.Optional; /** * Receives the environments declared by workers, so an aggregator evaluates their assets without declaring them itself. @@ -42,7 +40,7 @@ * two workers declaring the same asset id declare two assets. Only an {@link NodeMode#AGGREGATOR} serves it. */ @Path("api/workers") -@EndpointDisabled(name = NodeModeConfigSourceFactory.AGGREGATOR, stringValue = "false") +@EndpointDisabled(name = NodeModeConfigSourceFactory.AGGREGATOR_CONFIG_KEY, stringValue = "false") public class WorkerController { private final EnvironmentReconciler reconciler; diff --git a/src/main/java/tech/illuin/wombat/engine/connector/boavizta/BoaviztaConfig.java b/src/main/java/tech/illuin/wombat/engine/connector/boavizta/BoaviztaConfig.java index bc5f9ab..83e42c2 100644 --- a/src/main/java/tech/illuin/wombat/engine/connector/boavizta/BoaviztaConfig.java +++ b/src/main/java/tech/illuin/wombat/engine/connector/boavizta/BoaviztaConfig.java @@ -7,6 +7,7 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Singleton; import jakarta.ws.rs.Produces; +import tech.illuin.wombat.commons.WombatProperties; import tech.illuin.wombat.core.connector.boavizta.connector.BoaviztaClient; import tech.illuin.wombat.core.connector.boavizta.impact.BoaviztaEvaluationResolver; @@ -18,19 +19,21 @@ public class BoaviztaConfig { @Produces @Singleton - public BoaviztaClient provideBoaviztaClient(BoaviztaProperties properties) + public BoaviztaClient provideBoaviztaClient(WombatProperties wombat) { + BoaviztaProperties properties = wombat.connector().boavizta(); Feign.Builder builder = Feign.builder() .encoder(new JacksonEncoder()) .decoder(new JacksonDecoder()); - if (properties.proxyUrl().isPresent() && !properties.proxyUrl().get().isBlank()) - { - URI proxy = URI.create(properties.proxyUrl().get()); + properties.proxyUrl().ifPresent(proxyUrl -> { + if (proxyUrl.isBlank()) + return; + URI proxy = URI.create(proxyUrl); if (proxy.getHost() == null || proxy.getPort() == -1) - throw new IllegalArgumentException("connector.boavizta.proxy-url must include a host and port, got: " + properties.proxyUrl().get()); + throw new IllegalArgumentException("wombat.connector.boavizta.proxy-url must include a host and port, got: " + proxyUrl); builder.client(new Client.Proxied(null, null, new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxy.getHost(), proxy.getPort())))); - } + }); return builder.target(BoaviztaClient.class, properties.uri()); } diff --git a/src/main/java/tech/illuin/wombat/engine/connector/boavizta/BoaviztaProperties.java b/src/main/java/tech/illuin/wombat/engine/connector/boavizta/BoaviztaProperties.java index b1f2c1f..3a7bb34 100644 --- a/src/main/java/tech/illuin/wombat/engine/connector/boavizta/BoaviztaProperties.java +++ b/src/main/java/tech/illuin/wombat/engine/connector/boavizta/BoaviztaProperties.java @@ -1,10 +1,7 @@ package tech.illuin.wombat.engine.connector.boavizta; -import io.smallrye.config.ConfigMapping; - import java.util.Optional; -@ConfigMapping(prefix = "connector.boavizta") public interface BoaviztaProperties { String uri(); diff --git a/src/main/java/tech/illuin/wombat/engine/connector/ecologits/EcologitsConfig.java b/src/main/java/tech/illuin/wombat/engine/connector/ecologits/EcologitsConfig.java index 646e01c..a36250f 100644 --- a/src/main/java/tech/illuin/wombat/engine/connector/ecologits/EcologitsConfig.java +++ b/src/main/java/tech/illuin/wombat/engine/connector/ecologits/EcologitsConfig.java @@ -7,6 +7,7 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Singleton; import jakarta.ws.rs.Produces; +import tech.illuin.wombat.commons.WombatProperties; import tech.illuin.wombat.core.connector.ecologits.connector.EcologitsClient; import java.net.InetSocketAddress; @@ -17,19 +18,21 @@ public class EcologitsConfig { @Produces @Singleton - public EcologitsClient provideEcologitsClient(EcologitsProperties properties) + public EcologitsClient provideEcologitsClient(WombatProperties wombat) { + EcologitsProperties properties = wombat.connector().ecologits(); Feign.Builder builder = Feign.builder() .encoder(new JacksonEncoder()) .decoder(new JacksonDecoder()); - if (properties.proxyUrl().isPresent() && !properties.proxyUrl().get().isBlank()) - { - URI proxy = URI.create(properties.proxyUrl().get()); + properties.proxyUrl().ifPresent(proxyUrl -> { + if (proxyUrl.isBlank()) + return; + URI proxy = URI.create(proxyUrl); if (proxy.getHost() == null || proxy.getPort() == -1) - throw new IllegalArgumentException("connector.ecologits.proxy-url must include a host and port, got: " + properties.proxyUrl().get()); + throw new IllegalArgumentException("wombat.connector.ecologits.proxy-url must include a host and port, got: " + proxyUrl); builder.client(new Client.Proxied(null, null, new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxy.getHost(), proxy.getPort())))); - } + }); return builder.target(EcologitsClient.class, properties.uri()); } diff --git a/src/main/java/tech/illuin/wombat/engine/connector/ecologits/EcologitsProperties.java b/src/main/java/tech/illuin/wombat/engine/connector/ecologits/EcologitsProperties.java index 41c25b9..f783c0a 100644 --- a/src/main/java/tech/illuin/wombat/engine/connector/ecologits/EcologitsProperties.java +++ b/src/main/java/tech/illuin/wombat/engine/connector/ecologits/EcologitsProperties.java @@ -1,10 +1,7 @@ package tech.illuin.wombat.engine.connector.ecologits; -import io.smallrye.config.ConfigMapping; - import java.util.Optional; -@ConfigMapping(prefix = "connector.ecologits") public interface EcologitsProperties { String uri(); diff --git a/src/main/java/tech/illuin/wombat/engine/evaluation/CachedAssetEvaluatorProperties.java b/src/main/java/tech/illuin/wombat/engine/evaluation/CachedAssetEvaluatorProperties.java index ab98aa1..b5383eb 100644 --- a/src/main/java/tech/illuin/wombat/engine/evaluation/CachedAssetEvaluatorProperties.java +++ b/src/main/java/tech/illuin/wombat/engine/evaluation/CachedAssetEvaluatorProperties.java @@ -1,13 +1,11 @@ package tech.illuin.wombat.engine.evaluation; -import io.smallrye.config.ConfigMapping; import io.smallrye.config.WithDefault; import io.smallrye.config.WithName; import java.time.Duration; import java.util.List; -@ConfigMapping(prefix = "evaluation.cache") public interface CachedAssetEvaluatorProperties { @WithDefault("true") diff --git a/src/main/java/tech/illuin/wombat/engine/module/WombatCoreConfig.java b/src/main/java/tech/illuin/wombat/engine/module/WombatCoreConfig.java index 8bb66b6..aaa4b66 100644 --- a/src/main/java/tech/illuin/wombat/engine/module/WombatCoreConfig.java +++ b/src/main/java/tech/illuin/wombat/engine/module/WombatCoreConfig.java @@ -8,6 +8,7 @@ import tech.illuin.wombat.core.context.WombatContextProvider; import tech.illuin.wombat.core.evaluation.AssetEvaluator; import tech.illuin.wombat.core.module.WombatModule; +import tech.illuin.wombat.commons.WombatProperties; import tech.illuin.wombat.core.source.persistence.WombatMetricPersister; import tech.illuin.wombat.engine.evaluation.CachedAssetEvaluator; import tech.illuin.wombat.engine.evaluation.CachedAssetEvaluatorProperties; @@ -52,8 +53,9 @@ public WombatCore provideWombatCore( * as their type is already wired; a brand-new asset-type needs a fresh one. */ @Singleton - public AssetEvaluator provideImpactCalculator(WombatCore core, CachedAssetEvaluatorProperties cacheProperties) + public AssetEvaluator provideImpactCalculator(WombatCore core, WombatProperties wombat) { + CachedAssetEvaluatorProperties cacheProperties = wombat.evaluator().cache(); if (cacheProperties.enabled()) return new CachedAssetEvaluator(core.createEvaluator(), cacheProperties); return core.createEvaluator(); diff --git a/src/main/java/tech/illuin/wombat/engine/module/WombatModuleConfig.java b/src/main/java/tech/illuin/wombat/engine/module/WombatModuleConfig.java index 00fb43c..199b3ed 100644 --- a/src/main/java/tech/illuin/wombat/engine/module/WombatModuleConfig.java +++ b/src/main/java/tech/illuin/wombat/engine/module/WombatModuleConfig.java @@ -32,6 +32,7 @@ import tech.illuin.wombat.core.evaluation.impact.kubernetes.KubernetesMetricResolver; import tech.illuin.wombat.core.evaluation.impact.llm.LLMMetricResolver; import tech.illuin.wombat.core.module.WombatModule; +import tech.illuin.wombat.commons.WombatProperties; import tech.illuin.wombat.engine.module.extension.CompositeExtensionLoader; import tech.illuin.wombat.engine.module.extension.CoreExtensionLoader; import tech.illuin.wombat.engine.module.extension.DynamicExtensionLoader; @@ -77,11 +78,11 @@ public static DefaultHandlers provideLLMDefaults(LLMMetricResolver metricResolve } @Produces @Singleton - public static ExtensionLoader provideWombatExtensionLoader(WombatModuleProperties properties) + public static ExtensionLoader provideWombatExtensionLoader(WombatProperties wombat) { return new CompositeExtensionLoader( new CoreExtensionLoader(), - new DynamicExtensionLoader(properties.extension().path()) + new DynamicExtensionLoader(wombat.module().extension().path()) ); } @@ -108,8 +109,9 @@ public static List provideWombatModules(ExtensionLoader extensionL } @Produces @Singleton - public static ModuleRegistry provideModuleRegistry(List modules, WombatModuleProperties properties) + public static ModuleRegistry provideModuleRegistry(List modules, WombatProperties wombat) { + WombatModuleProperties properties = wombat.module(); /* This will soon become DB-backed and handle all detected modules; for now we use the conf-based switches for core modules only */ InMemoryModuleRegistry registry = new InMemoryModuleRegistry(); modules.forEach(module -> { diff --git a/src/main/java/tech/illuin/wombat/engine/module/extension/WombatModuleProperties.java b/src/main/java/tech/illuin/wombat/engine/module/extension/WombatModuleProperties.java index 4118c3d..5ef822f 100644 --- a/src/main/java/tech/illuin/wombat/engine/module/extension/WombatModuleProperties.java +++ b/src/main/java/tech/illuin/wombat/engine/module/extension/WombatModuleProperties.java @@ -1,12 +1,10 @@ package tech.illuin.wombat.engine.module.extension; -import io.smallrye.config.ConfigMapping; import io.smallrye.config.WithDefault; import io.smallrye.config.WithName; import java.nio.file.Path; -@ConfigMapping(prefix = "module") public interface WombatModuleProperties { @WithName("kubernetes-api") diff --git a/src/main/java/tech/illuin/wombat/engine/monitor/MonitorConfig.java b/src/main/java/tech/illuin/wombat/engine/monitor/MonitorConfig.java index 52a1158..fc65712 100644 --- a/src/main/java/tech/illuin/wombat/engine/monitor/MonitorConfig.java +++ b/src/main/java/tech/illuin/wombat/engine/monitor/MonitorConfig.java @@ -6,6 +6,7 @@ import jakarta.validation.ConstraintViolation; import jakarta.validation.ConstraintViolationException; import jakarta.validation.Validator; +import tech.illuin.wombat.commons.WombatProperties; import tech.illuin.wombat.core.WombatCore; import tech.illuin.wombat.core.source.AssetMonitor; @@ -19,9 +20,9 @@ public class MonitorConfig { @Singleton - public MonitoredEnvironments provideMonitoredEnvironments(MonitorProperties properties, YAMLMapper mapper, Validator validator) + public MonitoredEnvironments provideMonitoredEnvironments(WombatProperties wombat, YAMLMapper mapper, Validator validator) { - String location = properties.environmentsFile(); + String location = wombat.monitor().environmentsFile(); try (InputStream in = open(location)) { MonitoredEnvironments monitoredEnvironments = mapper.readValue(in, MonitoredEnvironments.class); diff --git a/src/main/java/tech/illuin/wombat/engine/monitor/MonitorProperties.java b/src/main/java/tech/illuin/wombat/engine/monitor/MonitorProperties.java index 1b0dcc6..eea4968 100644 --- a/src/main/java/tech/illuin/wombat/engine/monitor/MonitorProperties.java +++ b/src/main/java/tech/illuin/wombat/engine/monitor/MonitorProperties.java @@ -1,11 +1,9 @@ package tech.illuin.wombat.engine.monitor; -import io.smallrye.config.ConfigMapping; import io.smallrye.config.WithDefault; import io.smallrye.config.WithName; import tech.illuin.wombat.core.source.Period; -@ConfigMapping(prefix = "monitor") public interface MonitorProperties { Period heartbeat(); diff --git a/src/main/java/tech/illuin/wombat/engine/monitor/MonitorScheduler.java b/src/main/java/tech/illuin/wombat/engine/monitor/MonitorScheduler.java index 8989875..37cd53d 100644 --- a/src/main/java/tech/illuin/wombat/engine/monitor/MonitorScheduler.java +++ b/src/main/java/tech/illuin/wombat/engine/monitor/MonitorScheduler.java @@ -16,27 +16,23 @@ public class MonitorScheduler { private final Scheduler scheduler; private final Instance service; - private final MonitorProperties properties; private final NodeMode mode; + private final MonitorProperties properties; private static final Logger logger = LoggerFactory.getLogger(MonitorScheduler.class); @Inject - public MonitorScheduler( - Scheduler scheduler, - Instance service, - MonitorProperties properties, - WombatProperties wombat - ) { + public MonitorScheduler(Scheduler scheduler, Instance service, WombatProperties wombat) + { this.scheduler = scheduler; this.service = service; - this.properties = properties; this.mode = wombat.mode(); + this.properties = wombat.monitor(); } public void registerMonitorService(@Observes StartupEvent event) { - if (this.mode == NodeMode.AGGREGATOR) + if (this.mode == NodeMode.aggregator) { logger.info("Running in aggregator mode, environments are sampled by workers: skipping monitor service scheduling"); return; diff --git a/src/main/java/tech/illuin/wombat/persistence/BackendMode.java b/src/main/java/tech/illuin/wombat/persistence/BackendMode.java new file mode 100644 index 0000000..a9465ce --- /dev/null +++ b/src/main/java/tech/illuin/wombat/persistence/BackendMode.java @@ -0,0 +1,23 @@ +package tech.illuin.wombat.persistence; + +public enum BackendMode +{ + sqlite + { + @Override + public PersistenceBackendProperties.SQLiteProperties getSection(PersistenceBackendProperties parent) + { + return parent.sqlite().orElseThrow(() -> new IllegalArgumentException("sqlite properties required when mode=sqlite")); + } + }, + postgresql + { + @Override + public PersistenceBackendProperties.PostgreSQLProperties getSection(PersistenceBackendProperties parent) + { + return parent.postgresql().orElseThrow(() -> new IllegalArgumentException("postgresql properties required when mode=postgresql")); + } + }; + + public abstract T getSection(PersistenceBackendProperties parent); +} diff --git a/src/main/java/tech/illuin/wombat/persistence/PersistenceBackendConfigSourceFactory.java b/src/main/java/tech/illuin/wombat/persistence/PersistenceBackendConfigSourceFactory.java new file mode 100644 index 0000000..30716c6 --- /dev/null +++ b/src/main/java/tech/illuin/wombat/persistence/PersistenceBackendConfigSourceFactory.java @@ -0,0 +1,71 @@ +package tech.illuin.wombat.persistence; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigSourceContext; +import io.smallrye.config.ConfigSourceFactory; +import io.smallrye.config.PropertiesConfigSource; +import org.eclipse.microprofile.config.spi.ConfigSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +@StaticInitSafe +public class PersistenceBackendConfigSourceFactory implements ConfigSourceFactory +{ + public static final String ACTIVE_BACKEND_CONFIG_KEY = "_internal.persistence.backend"; + private static final Logger logger = LoggerFactory.getLogger(PersistenceBackendConfigSourceFactory.class); + + @Override + public Iterable getConfigSources(ConfigSourceContext context) + { + String modeStr = context.getValue("persistence.backend.mode").getValue(); + BackendMode mode = modeStr == null ? BackendMode.sqlite : parse(modeStr); + + Map properties = new HashMap<>(); + properties.put(ACTIVE_BACKEND_CONFIG_KEY, mode.name().toLowerCase(Locale.ROOT)); + + switch (mode) + { + case sqlite: + properties.put("quarkus.datasource.db-kind", "sqlite"); + String sqliteUrl = context.getValue("persistence.backend.sqlite.jdbc-url").getValue(); + if (sqliteUrl != null) + properties.put("quarkus.datasource.jdbc.url", sqliteUrl); + properties.put("quarkus.flyway.locations", "classpath:db/migration/sqlite"); + break; + + case postgresql: + properties.put("quarkus.datasource.db-kind", "postgresql"); + String pgUrl = context.getValue("persistence.backend.postgresql.jdbc-url").getValue(); + if (pgUrl != null) + properties.put("quarkus.datasource.jdbc.url", pgUrl); + String pgUser = context.getValue("persistence.backend.postgresql.username").getValue(); + if (pgUser != null) + properties.put("quarkus.datasource.username", pgUser); + String pgPass = context.getValue("persistence.backend.postgresql.password").getValue(); + if (pgPass != null) + properties.put("quarkus.datasource.password", pgPass); + properties.put("quarkus.flyway.locations", "classpath:db/migration/postgresql"); + break; + + default: + throw new IllegalArgumentException("Unknown backend mode: " + mode); + } + + logger.info("Persistence backend set to {} (priority {})", mode, Integer.MAX_VALUE - 1000); + return List.of(new PropertiesConfigSource( + properties, + PersistenceBackendConfigSourceFactory.class.getSimpleName(), + Integer.MAX_VALUE - 1000 + )); + } + + private static BackendMode parse(String mode) + { + return BackendMode.valueOf(mode.trim().replace('-', '_').toLowerCase(Locale.ROOT)); + } +} diff --git a/src/main/java/tech/illuin/wombat/persistence/PersistenceBackendProperties.java b/src/main/java/tech/illuin/wombat/persistence/PersistenceBackendProperties.java new file mode 100644 index 0000000..22f032d --- /dev/null +++ b/src/main/java/tech/illuin/wombat/persistence/PersistenceBackendProperties.java @@ -0,0 +1,35 @@ +package tech.illuin.wombat.persistence; + +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; +import tech.illuin.wombat.persistence.backend.api.backup.BackupProperties; + +import java.util.Optional; + +@ConfigMapping(prefix = "persistence.backend") +public interface PersistenceBackendProperties +{ + @WithDefault("sqlite") + BackendMode mode(); + + Optional sqlite(); + + Optional postgresql(); + + interface SQLiteProperties + { + String jdbcUrl(); + + Optional backup(); + } + + interface PostgreSQLProperties + { + Optional jdbcUrl(); + + @WithDefault("wombat") + Optional username(); + + Optional password(); + } +} diff --git a/src/main/java/tech/illuin/wombat/persistence/PersistenceConfig.java b/src/main/java/tech/illuin/wombat/persistence/PersistenceConfig.java index ca5b0cc..b405ed0 100644 --- a/src/main/java/tech/illuin/wombat/persistence/PersistenceConfig.java +++ b/src/main/java/tech/illuin/wombat/persistence/PersistenceConfig.java @@ -3,7 +3,6 @@ import io.micrometer.core.instrument.MeterRegistry; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Singleton; -import org.eclipse.microprofile.config.inject.ConfigProperty; import tech.illuin.wombat.core.source.persistence.WombatMetricPersister; import tech.illuin.wombat.core.source.persistence.micrometer.MicrometerMetricPersister; import tech.illuin.wombat.persistence.dialect.JsonPathDialect; @@ -11,9 +10,6 @@ @ApplicationScoped public class PersistenceConfig { - private static final String DBKIND_SQLITE = "sqlite"; - private static final String DBKIND_POSTGRESQL = "postgresql"; - @Singleton public WombatMetricPersister provideMicrometerMetricPersister(MeterRegistry meterRegistry) { @@ -21,13 +17,12 @@ public WombatMetricPersister provideMicrometerMetricPersister(MeterRegistry mete } @Singleton - public JsonPathDialect provideJsonPathDialect(@ConfigProperty(name = "quarkus.datasource.db-kind") String dbKind) + public JsonPathDialect provideJsonPathDialect(PersistenceBackendProperties backendProps) { - return switch (dbKind) + return switch (backendProps.mode()) { - case DBKIND_SQLITE -> JsonPathDialect.SQLITE; - case DBKIND_POSTGRESQL -> JsonPathDialect.POSTGRESQL; - default -> throw new IllegalArgumentException("No JSON path dialect for db-kind: " + dbKind); + case sqlite -> JsonPathDialect.SQLITE; + case postgresql -> JsonPathDialect.POSTGRESQL; }; } } diff --git a/src/main/java/tech/illuin/wombat/persistence/backend/api/backup/BackupProperties.java b/src/main/java/tech/illuin/wombat/persistence/backend/api/backup/BackupProperties.java index 580e797..0e3babf 100644 --- a/src/main/java/tech/illuin/wombat/persistence/backend/api/backup/BackupProperties.java +++ b/src/main/java/tech/illuin/wombat/persistence/backend/api/backup/BackupProperties.java @@ -6,7 +6,7 @@ import java.util.Optional; -@ConfigMapping(prefix = "backup") +@ConfigMapping(prefix = "persistence.backend.sqlite.backup") public interface BackupProperties { @WithDefault("false") diff --git a/src/main/java/tech/illuin/wombat/persistence/backend/postgres/PostgresBackendConfig.java b/src/main/java/tech/illuin/wombat/persistence/backend/postgres/PostgresBackendConfig.java index d395e10..1947ae7 100644 --- a/src/main/java/tech/illuin/wombat/persistence/backend/postgres/PostgresBackendConfig.java +++ b/src/main/java/tech/illuin/wombat/persistence/backend/postgres/PostgresBackendConfig.java @@ -6,8 +6,9 @@ import jakarta.enterprise.inject.Instance; import jakarta.inject.Singleton; import jakarta.ws.rs.Produces; -import org.eclipse.microprofile.config.inject.ConfigProperty; import org.flywaydb.core.Flyway; +import tech.illuin.wombat.persistence.BackendMode; +import tech.illuin.wombat.persistence.PersistenceBackendProperties; import tech.illuin.wombat.persistence.backend.api.*; import tech.illuin.wombat.persistence.backend.api.store.MetricStoreContainer; import tech.illuin.wombat.persistence.backend.api.action.FlywayMigrate; @@ -25,13 +26,13 @@ public class PostgresBackendConfig @Produces @Singleton public PersistenceBackend providePostgresBackend( - @ConfigProperty(name = "quarkus.datasource.db-kind") String dbKind, + PersistenceBackendProperties backendProps, Instance dataSources, Instance flyway, Instance registry, MetricStoreContainer databaseStores ) { - if (!dbKind.equals(DBKIND_POSTGRESQL)) + if (backendProps.mode() != BackendMode.postgresql) return PersistenceBackend.disabled(DBKIND_POSTGRESQL); List hooks = new ArrayList<>(); diff --git a/src/main/java/tech/illuin/wombat/persistence/backend/push/PushBackendConfig.java b/src/main/java/tech/illuin/wombat/persistence/backend/push/PushBackendConfig.java index 00e542f..afe53cd 100644 --- a/src/main/java/tech/illuin/wombat/persistence/backend/push/PushBackendConfig.java +++ b/src/main/java/tech/illuin/wombat/persistence/backend/push/PushBackendConfig.java @@ -52,7 +52,7 @@ public PersistenceBackend providePushBackend( Instance assetMapper, Instance environments ) { - if (wombat.mode() != NodeMode.WORKER) + if (wombat.mode() != NodeMode.worker) return PersistenceBackend.disabled(BACKEND_PUSH); WorkerProperties properties = wombat.worker(); @@ -90,13 +90,14 @@ public static T createClient(Class type, AggregatorProperties properties, .requestInterceptor(template -> template.header(ApiKeyAuthenticationMechanism.HEADER, properties.apiKey())) .options(new Request.Options(timeout, timeout, true)); - if (properties.proxyUrl().isPresent() && !properties.proxyUrl().get().isBlank()) - { - URI proxy = URI.create(properties.proxyUrl().get()); + properties.proxyUrl().ifPresent(proxyUrl -> { + if (proxyUrl.isBlank()) + return; + URI proxy = URI.create(proxyUrl); if (proxy.getHost() == null || proxy.getPort() == -1) - throw new IllegalArgumentException("wombat.worker.aggregator.proxy-url must include a host and port, got: " + properties.proxyUrl().get()); + throw new IllegalArgumentException("wombat.worker.aggregator.proxy-url must include a host and port, got: " + proxyUrl); builder.client(new Client.Proxied(null, null, new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxy.getHost(), proxy.getPort())))); - } + }); return builder.target(type, properties.url().replaceAll("/+$", "")); } diff --git a/src/main/java/tech/illuin/wombat/persistence/backend/sqlite/SQLiteBackendConfig.java b/src/main/java/tech/illuin/wombat/persistence/backend/sqlite/SQLiteBackendConfig.java index 744d7b2..62be549 100644 --- a/src/main/java/tech/illuin/wombat/persistence/backend/sqlite/SQLiteBackendConfig.java +++ b/src/main/java/tech/illuin/wombat/persistence/backend/sqlite/SQLiteBackendConfig.java @@ -6,14 +6,16 @@ import jakarta.enterprise.inject.Instance; import jakarta.inject.Singleton; import jakarta.ws.rs.Produces; -import org.eclipse.microprofile.config.inject.ConfigProperty; import org.flywaydb.core.Flyway; +import tech.illuin.wombat.persistence.BackendMode; +import tech.illuin.wombat.persistence.PersistenceBackendProperties; +import tech.illuin.wombat.persistence.PersistenceBackendProperties.SQLiteProperties; import tech.illuin.wombat.persistence.backend.api.*; import tech.illuin.wombat.persistence.backend.api.store.MetricStoreContainer; import tech.illuin.wombat.persistence.backend.api.action.FlywayMigrate; +import tech.illuin.wombat.persistence.backend.api.backup.BackupProperties; import tech.illuin.wombat.persistence.backend.s3.S3Properties; import tech.illuin.wombat.persistence.backend.sqlite.action.*; -import tech.illuin.wombat.persistence.backend.api.backup.BackupProperties; import java.nio.file.Path; import java.util.ArrayList; @@ -29,17 +31,18 @@ public class SQLiteBackendConfig @Produces @Singleton public PersistenceBackend provideSQLiteBackend( - @ConfigProperty(name = "quarkus.datasource.db-kind") String dbKind, - @ConfigProperty(name = "quarkus.datasource.jdbc.url") String jdbcUrl, - BackupProperties backupProperties, + PersistenceBackendProperties backendProps, Instance dataSources, Instance flyway, Instance registry, MetricStoreContainer databaseStores ) { - if (!dbKind.equals(DBKIND_SQLITE)) + if (backendProps.mode() != BackendMode.sqlite) return PersistenceBackend.disabled(DBKIND_SQLITE); + SQLiteProperties sqlite = backendProps.mode().getSection(backendProps); + String jdbcUrl = sqlite.jdbcUrl(); + List hooks = new ArrayList<>(); List processes = new ArrayList<>(); @@ -51,11 +54,10 @@ public PersistenceBackend provideSQLiteBackend( if (!jdbcPath.equals(":memory:")) { Path dbPath = Path.of(jdbcPath); - + hooks.add(new HookSupplier("directory-init", RESOURCE_INIT, 0, () -> new SQLiteDirectoryInit(dbPath))); - if (backupProperties.enabled()) - { + sqlite.backup().filter(BackupProperties::enabled).ifPresent(backupProperties -> { S3Properties s3Properties = backupProperties.s3().orElseThrow(() -> new IllegalArgumentException("S3 properties must be provided when backup is enabled")); Supplier backupAction = () -> new SQLiteBackupProduce(dataSources.get(), s3Properties); @@ -66,7 +68,7 @@ public PersistenceBackend provideSQLiteBackend( hooks.add(new HookSupplier("backup-restore", RESOURCE_INIT, 16, () -> new SQLiteBackupRestore(dbPath, s3Properties))); if (backupProperties.cleanup().enabled()) processes.add(new ProcessSupplier("backup-clean", backupProperties.cleanup().cron(), () -> new SQLiteBackupClean(backupProperties.cleanup(), s3Properties))); - } + }); } return PersistenceBackend.of(DBKIND_SQLITE, hooks, processes, databaseStores); diff --git a/src/main/java/tech/illuin/wombat/persistence/backend/sqlite/SQLiteProperties.java b/src/main/java/tech/illuin/wombat/persistence/backend/sqlite/SQLiteProperties.java deleted file mode 100644 index 53241a3..0000000 --- a/src/main/java/tech/illuin/wombat/persistence/backend/sqlite/SQLiteProperties.java +++ /dev/null @@ -1,11 +0,0 @@ -package tech.illuin.wombat.persistence.backend.sqlite; - -import tech.illuin.wombat.persistence.backend.api.backup.BackupProperties; - -import java.nio.file.Path; - -public record SQLiteProperties( - String jdbcUrl, - Path dbPath, - BackupProperties backup -) {} diff --git a/src/main/java/tech/illuin/wombat/persistence/compaction/CompactionProperties.java b/src/main/java/tech/illuin/wombat/persistence/compaction/CompactionProperties.java index ed7fc19..38a0e0f 100644 --- a/src/main/java/tech/illuin/wombat/persistence/compaction/CompactionProperties.java +++ b/src/main/java/tech/illuin/wombat/persistence/compaction/CompactionProperties.java @@ -5,7 +5,7 @@ import java.time.Duration; -@ConfigMapping(prefix = "compaction") +@ConfigMapping(prefix = "persistence.compaction") public interface CompactionProperties { @WithDefault("0 1 * * * ?") diff --git a/src/main/java/tech/illuin/wombat/persistence/compaction/CompactionService.java b/src/main/java/tech/illuin/wombat/persistence/compaction/CompactionService.java index 57ba385..cdb39bb 100644 --- a/src/main/java/tech/illuin/wombat/persistence/compaction/CompactionService.java +++ b/src/main/java/tech/illuin/wombat/persistence/compaction/CompactionService.java @@ -13,7 +13,7 @@ public CompactionService(MetricCompactor compactor) this.compactor = compactor; } - @Scheduled(cron = "${compaction.cron}") + @Scheduled(cron = "${persistence.compaction.cron}") public void compact() { this.compactor.compact(Instant.now()); diff --git a/src/main/java/tech/illuin/wombat/persistence/micrometer/MetricsConfig.java b/src/main/java/tech/illuin/wombat/persistence/micrometer/MetricsConfig.java index 18eebf3..a1c9286 100644 --- a/src/main/java/tech/illuin/wombat/persistence/micrometer/MetricsConfig.java +++ b/src/main/java/tech/illuin/wombat/persistence/micrometer/MetricsConfig.java @@ -12,9 +12,9 @@ import org.jspecify.annotations.NonNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.eclipse.microprofile.config.inject.ConfigProperty; import tech.illuin.wombat.commons.NodeMode; import tech.illuin.wombat.commons.WombatProperties; +import tech.illuin.wombat.persistence.PersistenceBackendProperties; import tech.illuin.wombat.persistence.backend.PersistenceEngine; import tech.illuin.wombat.persistence.backend.push.PushBackendConfig; import tech.illuin.wombat.persistence.backend.api.store.MetricStoreContainer; @@ -32,15 +32,14 @@ public class MetricsConfig public WombatStepMeterRegistry provideStepRegistry( PersistenceEngine engine, WombatProperties wombat, - @ConfigProperty(name = "quarkus.datasource.db-kind") String dbKind + PersistenceBackendProperties backendProps ) { NodeMode mode = wombat.mode(); - String backend = mode == NodeMode.WORKER ? PushBackendConfig.BACKEND_PUSH : dbKind; + String backend = mode == NodeMode.worker ? PushBackendConfig.BACKEND_PUSH : backendProps.mode().name(); MetricStoreContainer stores = engine.stores(backend); logger.info("Running in {} mode, sampled metrics are written to the {} backend", mode, backend); - StepRegistryConfig stepConfig = new StepRegistryConfig() - { + StepRegistryConfig stepConfig = new StepRegistryConfig() { @Override public String prefix() { diff --git a/src/main/resources/application-dev.yaml.dist b/src/main/resources/application-dev.yaml.dist index 9595270..fbd0ec2 100644 --- a/src/main/resources/application-dev.yaml.dist +++ b/src/main/resources/application-dev.yaml.dist @@ -1,55 +1,55 @@ -quarkus: +#quarkus: # dev inherits the oidc profile (application-oidc.yaml and the "%oidc" block below), so `quarkus:dev` requires signing in. # Uncomment to run with authentication. #config: # profile: # parent: oidc # Alternative PostgreSQL Datasource for dev: - # datasource: - # db-kind: postgresql - # username: wombat - # password: wombat - # jdbc: - # url: "jdbc:postgresql://localhost:5432/wombat" + # Database configuration is handled dynamically by PersistenceBackendConfigSourceFactory + # based on persistence.backend.mode — do not set quarkus.datasource, hibernate-orm, or flyway here. #vault: # devservices: # enabled: true # kv-secret-engine-mount-path: secret -# Connector endpoints for local mock / stub services -connector: - boavizta: - uri: "http://localhost:5001/v1/" - ecologits: - uri: "http://localhost:5002/" +wombat: + # Connector endpoints for local mock / stub services + connector: + boavizta: + uri: "http://localhost:5001/v1/" + ecologits: + uri: "http://localhost:5002/" -monitor: - environments-file: monitored/monitored-environments-dev.yaml + monitor: + environments-file: monitored/monitored-environments-dev.yaml -# Fold sampling rows every 5 minutes instead of every hour, so fresh activity reaches the UI within minutes -# Keep the defaults in production, see doc/compaction.md -compaction: - cron: "0 1/5 * * * ?" # a minute past every 5-minute window - window: 5m - delay: 30s + # Optional Secret resolution directory: + #secret: + # directory: + # path: /secrets -# Local S3 Backup setup (e.g. SeaweedFS service), only used by the SQLite backend: drop it when running on PostgreSQL -# backup: -# enabled: true -# cron: "0 0 * * * ?" # hourly -# s3: -# key-prefix: sqlite/ -# endpoint: http://localhost:8333 -# bucket: wombat-bucket -# region: us-east-1 -# access-key: admin -# secret-key: secret +persistence: + #backend: + # mode: sqlite + # sqlite: + #backup: + # enabled: true + # cron: "0 0 * * * ?" # hourly + ## Local S3 Backup setup (e.g. SeaweedFS service), only used by the SQLite backend: drop it when running on PostgreSQL + #s3: + # key-prefix: sqlite/ + # endpoint: http://localhost:8333 + # bucket: wombat-bucket + # region: us-east-1 + # access-key: admin + # secret-key: secret -# Optional Secret resolution directory: -# wombat: -# secret: -# directory: -# path: /secrets + # Fold sampling rows every 5 minutes instead of every hour, so fresh activity reaches the UI within minutes + # Keep the defaults in production, see doc/compaction.md + compaction: + cron: "0 1/5 * * * ?" # a minute past every 5-minute window + window: 5m + delay: 30s # Identity provider for dev mode, see doc/access-control.md. Register http://localhost:8080/auth/callback as a redirect URI. "%oidc": diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 9ffaf00..2a67b34 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -50,25 +50,18 @@ quarkus: path: /api/swagger always-include: true + # Database configuration is handled dynamically by PersistenceBackendConfigSourceFactory + # based on persistence.backend.mode. The values below are build-time defaults only + # and are overridden at runtime with priority Integer.MAX_VALUE - 1000. datasource: db-kind: sqlite jdbc: - url: "jdbc:sqlite:./data/db/local.db" - min-size: 0 - ## PostgreSQL Datasource Configuration - #datasource: - # db-kind: postgresql - # username: my-username - # password: my-password - # jdbc: - # url: "jdbc:postgresql://localhost:5432/wombat" - + url: "jdbc:sqlite::memory:" hibernate-orm: schema-management: strategy: none flyway: - migrate-at-start: false - locations: classpath:db/migration/${quarkus.datasource.db-kind} + locations: classpath:db/migration/sqlite scheduler: enabled: true kubernetes-client: @@ -99,44 +92,6 @@ quarkus: quinoa: enabled: false -module: - kubernetes-api: - enabled: true - llm-prometheus: - enabled: true - llm-anthropic: - enabled: true - extension: - path: "extensions" - -evaluation: - cache: - enabled: true - maximum-size: 5000 - expire-after-write: 10m - -compaction: - cron: "0 1 * * * ?" # a minute past every hour - window: 1h - delay: 1m - -monitor: - heartbeat: "10s" - environments-file: "monitored/monitored-environments.yaml" - -ui: - max-date-range: - duration: 1 - unit: YEARS - -connector: - boavizta: - uri: "https://api.boavizta.org/v1/" - #proxy-url: "http://proxy.example.com:3128" - ecologits: - uri: "https://api.ecologits.ai/" - #proxy-url: "http://proxy.example.com:3128" - wombat: # standalone: samples its environments into its own database (single-instance setup) # aggregator: stores and serves the metrics pushed by workers on POST /api/workers/{worker}/metrics, samples nothing itself @@ -152,6 +107,34 @@ wombat: # proxy-url: "http://proxy.example.com:3128" # optional, HTTP proxy to reach the aggregator through # timeout: 30s + connector: + boavizta: + uri: "https://api.boavizta.org/v1/" + #proxy-url: "http://proxy.example.com:3128" + ecologits: + uri: "https://api.ecologits.ai/" + #proxy-url: "http://proxy.example.com:3128" + + monitor: + heartbeat: "10s" + environments-file: "monitored/monitored-environments.yaml" + + evaluator: + cache: + enabled: true + maximum-size: 5000 + expire-after-write: 10m + + module: + kubernetes-api: + enabled: true + llm-prometheus: + enabled: true + llm-anthropic: + enabled: true + extension: + path: "extensions" + ## Secret Resolution Configuration # Write mode defaults to DISABLED; DATABASE is selected at startup and requires a restart to change. # Supply KEKs through external configuration (e.g. environment-variable @@ -169,19 +152,39 @@ wombat: # kek: # current: "${WOMBAT_SECRET_DATABASE_KEK_CURRENT}" -backup: - enabled: false - cron: "0 0 2 * * ?" - restore-on-startup: true - cleanup: - enabled: true - cron: "0 0 3 * * ?" - retain-last: 10 - ## S3 Backup Storage Configuration - #s3: - # endpoint: "https://s3.amazonaws.com" - # bucket: "wombat-backups" - # region: "us-east-1" - # key-prefix: "backups/" - # access-key: "your-access-key" - # secret-key: "your-secret-key" +persistence: + backend: + mode: sqlite + sqlite: + jdbc-url: "jdbc:sqlite:./data/db/local.db" + backup: + enabled: false + cron: "0 0 2 * * ?" + restore-on-startup: true + cleanup: + enabled: true + cron: "0 0 3 * * ?" + retain-last: 10 + ## S3 Backup Storage Configuration + #s3: + # endpoint: "https://s3.amazonaws.com" + # bucket: "wombat-backups" + # region: "us-east-1" + # key-prefix: "backups/" + # access-key: "your-access-key" + # secret-key: "your-secret-key" + ## PostgreSQL Configuration + #postgresql: + # jdbc-url: "jdbc:postgresql://localhost:5432/wombat" + # username: my-username + # password: my-password + + compaction: + cron: "0 1 * * * ?" # a minute past every hour + window: 1h + delay: 1m + +ui: + max-date-range: + duration: 1 + unit: YEARS diff --git a/src/test/java/tech/illuin/wombat/endpoint/DisabledModuleProfile.java b/src/test/java/tech/illuin/wombat/endpoint/DisabledModuleProfile.java index a4cfbf1..53bd093 100644 --- a/src/test/java/tech/illuin/wombat/endpoint/DisabledModuleProfile.java +++ b/src/test/java/tech/illuin/wombat/endpoint/DisabledModuleProfile.java @@ -12,7 +12,7 @@ public Map getConfigOverrides() { return Map.of( "wombat.mode", "aggregator", - "module.kubernetes-api.enabled", "false" + "wombat.module.kubernetes-api.enabled", "false" ); } } diff --git a/src/test/java/tech/illuin/wombat/engine/evaluation/CachedAssetEvaluatorPropertiesTest.java b/src/test/java/tech/illuin/wombat/engine/evaluation/CachedAssetEvaluatorPropertiesTest.java index 6da9908..0fa1216 100644 --- a/src/test/java/tech/illuin/wombat/engine/evaluation/CachedAssetEvaluatorPropertiesTest.java +++ b/src/test/java/tech/illuin/wombat/engine/evaluation/CachedAssetEvaluatorPropertiesTest.java @@ -26,10 +26,10 @@ void missingActivityResolverCachesDefaultsToAnEmptyList() void indexedActivityResolverCacheUsesItsModuleAndDefaultCacheSettings() { CachedAssetEvaluatorProperties properties = load(Map.of( - "evaluation.cache.activity-resolver-caches[0].module", "tech.custom:llm", - "evaluation.cache.activity-resolver-caches[1].module", "tech.custom:container", - "evaluation.cache.activity-resolver-caches[1].maximum-size", "25", - "evaluation.cache.activity-resolver-caches[1].expire-after-write", "2m" + "wombat.evaluator.cache.activity-resolver-caches[0].module", "tech.custom:llm", + "wombat.evaluator.cache.activity-resolver-caches[1].module", "tech.custom:container", + "wombat.evaluator.cache.activity-resolver-caches[1].maximum-size", "25", + "wombat.evaluator.cache.activity-resolver-caches[1].expire-after-write", "2m" )); assertEquals(2, properties.activityResolverCaches().size()); @@ -44,10 +44,10 @@ void indexedActivityResolverCacheUsesItsModuleAndDefaultCacheSettings() private static CachedAssetEvaluatorProperties load(Map values) { SmallRyeConfig config = new SmallRyeConfigBuilder() - .withMapping(CachedAssetEvaluatorProperties.class) - .withConverter(Duration.class, 100, new DurationConverter()) .withSources(new PropertiesConfigSource(values, "test", 100)) + .withMapping(CachedAssetEvaluatorProperties.class, "wombat.evaluator.cache") + .withConverter(Duration.class, 100, new DurationConverter()) .build(); - return config.getConfigMapping(CachedAssetEvaluatorProperties.class); + return config.getConfigMapping(CachedAssetEvaluatorProperties.class, "wombat.evaluator.cache"); } } \ No newline at end of file diff --git a/src/test/java/tech/illuin/wombat/engine/module/WombatModuleConfigTest.java b/src/test/java/tech/illuin/wombat/engine/module/WombatModuleConfigTest.java index 4404edf..a52a34d 100644 --- a/src/test/java/tech/illuin/wombat/engine/module/WombatModuleConfigTest.java +++ b/src/test/java/tech/illuin/wombat/engine/module/WombatModuleConfigTest.java @@ -10,6 +10,10 @@ import tech.illuin.wombat.core.evaluation.impact.kubernetes.KubernetesMetricResolver; import tech.illuin.wombat.core.evaluation.impact.llm.LLMMetricResolver; import tech.illuin.wombat.core.module.WombatModule; +import tech.illuin.wombat.commons.NodeMode; +import tech.illuin.wombat.commons.WombatProperties; +import tech.illuin.wombat.engine.connector.boavizta.BoaviztaProperties; +import tech.illuin.wombat.engine.connector.ecologits.EcologitsProperties; import tech.illuin.wombat.engine.module.library.kubernetes_simulated.KubernetesSimulatedModule; import tech.illuin.wombat.engine.module.library.llm_simulated.LLMSimulatedModule; import tech.illuin.wombat.engine.module.extension.CompositeExtensionLoader; @@ -18,6 +22,9 @@ import tech.illuin.wombat.engine.module.registry.ModuleMetadata; import tech.illuin.wombat.engine.module.registry.ModuleRegistries; import tech.illuin.wombat.engine.module.registry.ModuleRegistry; +import tech.illuin.wombat.persistence.BackendMode; +import tech.illuin.wombat.persistence.PersistenceBackendProperties; +import tech.illuin.wombat.persistence.backend.push.WorkerProperties; import tech.illuin.wombat.module.kubernetes_static.KubernetesStaticModule; import tech.illuin.wombat.module.llm_anthropic.LLMAnthropicModule; import tech.illuin.wombat.module.kubernetes_api.KubernetesAPIModule; @@ -27,6 +34,7 @@ import java.nio.file.Path; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,6 +43,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static tech.illuin.wombat.engine.module.WombatModuleConfig.*; /** * Exercises the producers directly rather than through CDI: the toggles are plain logic over @@ -48,8 +57,8 @@ class WombatModuleConfigTest @Test void provideWombatExtensionLoader_returnsCompositeExtensionLoader() { - WombatModuleProperties properties = properties(true, true, true, true); - ExtensionLoader loader = this.config.provideWombatExtensionLoader(properties); + WombatProperties wombat = wombatProperties(true, true, true, true); + ExtensionLoader loader = provideWombatExtensionLoader(wombat); assertInstanceOf(CompositeExtensionLoader.class, loader); } @@ -61,7 +70,7 @@ void provideWombatModules_loadsModulesFromExtensionLoader() List expected = List.of(new KubernetesAPIModule(), new LLMStaticModule()); when(loader.load()).thenReturn(expected); - List result = this.config.provideWombatModules(loader); + List result = provideWombatModules(loader); assertEquals(expected, result); verify(loader).load(); @@ -82,14 +91,14 @@ void provideModuleRegistry_usesConfigurationForEnabledMetadata() ); assertAll( - () -> assertEnabled(this.config.provideModuleRegistry(modules, properties(false, true, true, true)), new KubernetesAPIModule(), false), - () -> assertEnabled(this.config.provideModuleRegistry(modules, properties(true, false, true, true)), new LLMPrometheusModule(), false), - () -> assertEnabled(this.config.provideModuleRegistry(modules, properties(true, true, false, true)), new LLMAnthropicModule(), false), - () -> assertEnabled(this.config.provideModuleRegistry(modules, properties(true, true, true, false)), new LLMVertexAIModule(), false), - () -> assertEnabled(this.config.provideModuleRegistry(modules, properties(false, false, false, false)), new KubernetesSimulatedModule(), true), - () -> assertEnabled(this.config.provideModuleRegistry(modules, properties(false, false, false, false)), new LLMSimulatedModule(), true), - () -> assertEnabled(this.config.provideModuleRegistry(modules, properties(false, false, false, false)), new KubernetesStaticModule(), true), - () -> assertEnabled(this.config.provideModuleRegistry(modules, properties(false, false, false, false)), new LLMStaticModule(), true) + () -> assertEnabled(provideModuleRegistry(modules, wombatProperties(false, true, true, true)), new KubernetesAPIModule(), false), + () -> assertEnabled(provideModuleRegistry(modules, wombatProperties(true, false, true, true)), new LLMPrometheusModule(), false), + () -> assertEnabled(provideModuleRegistry(modules, wombatProperties(true, true, false, true)), new LLMAnthropicModule(), false), + () -> assertEnabled(provideModuleRegistry(modules, wombatProperties(true, true, true, false)), new LLMVertexAIModule(), false), + () -> assertEnabled(provideModuleRegistry(modules, wombatProperties(false, false, false, false)), new KubernetesSimulatedModule(), true), + () -> assertEnabled(provideModuleRegistry(modules, wombatProperties(false, false, false, false)), new LLMSimulatedModule(), true), + () -> assertEnabled(provideModuleRegistry(modules, wombatProperties(false, false, false, false)), new KubernetesStaticModule(), true), + () -> assertEnabled(provideModuleRegistry(modules, wombatProperties(false, false, false, false)), new LLMStaticModule(), true) ); } @@ -189,38 +198,64 @@ void provideDefaults_createsHandlersForFamilies() assertNotNull(llmHandlers.costResolver()); } - private static WombatModuleProperties properties(boolean kubernetesApi, boolean llmPrometheus, boolean llmAnthropic, boolean vertexAi) + private static WombatProperties wombatProperties(boolean kubernetesApi, boolean llmPrometheus, boolean llmAnthropic, boolean vertexAi) { - return new WombatModuleProperties() + return new WombatProperties() { @Override - public KubernetesApiProperties kubernetesApi() - { - return () -> kubernetesApi; - } + public NodeMode mode() { return NodeMode.standalone; } @Override - public LlmPrometheusProperties llmPrometheus() + public WorkerProperties worker() { - return () -> llmPrometheus; + return new WorkerProperties() { + @Override public Optional id() { return Optional.empty(); } + @Override public Optional aggregator() { return Optional.empty(); } + }; } @Override - public LlmAnthropicProperties llmAnthropic() + public tech.illuin.wombat.engine.monitor.MonitorProperties monitor() { return null; } + + @Override + public WombatProperties.EvaluatorProperties evaluator() { - return () -> llmAnthropic; + return () -> null; } @Override - public LlmVertexAIProperties llmVertexAi() + public WombatModuleProperties module() { - return () -> vertexAi; + return new WombatModuleProperties() + { + @Override + public WombatModuleProperties.KubernetesApiProperties kubernetesApi() { return () -> kubernetesApi; } + + @Override + public WombatModuleProperties.LlmPrometheusProperties llmPrometheus() { return () -> llmPrometheus; } + + @Override + public WombatModuleProperties.LlmAnthropicProperties llmAnthropic() { return () -> llmAnthropic; } + + @Override + public WombatModuleProperties.LlmVertexAIProperties llmVertexAi() { return () -> vertexAi; } + + @Override + public WombatModuleProperties.ExtensionProperties extension() { return () -> Path.of("extensions"); } + }; } @Override - public ExtensionProperties extension() + public WombatProperties.ConnectorProperties connector() { - return () -> Path.of("extensions"); + return new WombatProperties.ConnectorProperties() + { + @Override + public BoaviztaProperties boavizta() { return null; } + + @Override + public EcologitsProperties ecologits() { return null; } + }; } }; } diff --git a/src/test/java/tech/illuin/wombat/engine/monitor/MonitorSchedulerTest.java b/src/test/java/tech/illuin/wombat/engine/monitor/MonitorSchedulerTest.java index d15052f..ad0049e 100644 --- a/src/test/java/tech/illuin/wombat/engine/monitor/MonitorSchedulerTest.java +++ b/src/test/java/tech/illuin/wombat/engine/monitor/MonitorSchedulerTest.java @@ -19,11 +19,11 @@ class MonitorSchedulerTest @Test void standaloneAndWorkers_sampleTheirEnvironments() { - for (NodeMode mode : new NodeMode[]{ NodeMode.STANDALONE, NodeMode.WORKER }) + for (NodeMode mode : new NodeMode[]{ NodeMode.standalone, NodeMode.worker }) { Scheduler scheduler = startedScheduler(); Instance service = service(); - new MonitorScheduler(scheduler, service, properties(), wombat(mode)).registerMonitorService(null); + new MonitorScheduler(scheduler, service, wombat(mode)).registerMonitorService(null); verify(scheduler).newJob("wombat-monitor"); verify(service).get(); } @@ -35,7 +35,7 @@ void aggregator_leavesTheSamplingToItsWorkers() { Scheduler scheduler = startedScheduler(); Instance service = service(); - new MonitorScheduler(scheduler, service, properties(), wombat(NodeMode.AGGREGATOR)).registerMonitorService(null); + new MonitorScheduler(scheduler, service, wombat(NodeMode.aggregator)).registerMonitorService(null); verify(scheduler, never()).newJob(anyString()); verify(service, never()).get(); } @@ -58,13 +58,8 @@ private static Scheduler startedScheduler() private static WombatProperties wombat(NodeMode mode) { - WombatProperties wombat = mock(WombatProperties.class); + WombatProperties wombat = mock(WombatProperties.class, RETURNS_DEEP_STUBS); when(wombat.mode()).thenReturn(mode); return wombat; } - - private static MonitorProperties properties() - { - return mock(MonitorProperties.class, RETURNS_DEEP_STUBS); - } } diff --git a/src/test/java/tech/illuin/wombat/engine/monitor/MonitoredEnvironmentsTemplateTest.java b/src/test/java/tech/illuin/wombat/engine/monitor/MonitoredEnvironmentsTemplateTest.java index 5ddd04b..1e616c1 100644 --- a/src/test/java/tech/illuin/wombat/engine/monitor/MonitoredEnvironmentsTemplateTest.java +++ b/src/test/java/tech/illuin/wombat/engine/monitor/MonitoredEnvironmentsTemplateTest.java @@ -6,11 +6,17 @@ import jakarta.validation.Validation; import jakarta.validation.Validator; import org.junit.jupiter.api.Test; +import tech.illuin.wombat.commons.NodeMode; +import tech.illuin.wombat.commons.WombatProperties; +import tech.illuin.wombat.engine.connector.boavizta.BoaviztaProperties; +import tech.illuin.wombat.engine.connector.ecologits.EcologitsProperties; import tech.illuin.wombat.core.asset.Asset; import tech.illuin.wombat.core.asset.type.AssetType; import tech.illuin.wombat.core.secret.SecretAware; import tech.illuin.wombat.core.source.Period; +import tech.illuin.wombat.engine.evaluation.CachedAssetEvaluatorProperties; import tech.illuin.wombat.engine.module.WombatModuleConfig; +import tech.illuin.wombat.engine.module.extension.WombatModuleProperties; import tech.illuin.wombat.engine.module.registry.ModuleRegistries; import tech.illuin.wombat.module.kubernetes_api.KubernetesAPIAsset; import tech.illuin.wombat.module.kubernetes_api.KubernetesAPIModule; @@ -18,8 +24,12 @@ import tech.illuin.wombat.module.llm_prometheus.LLMPrometheusModule; import tech.illuin.wombat.module.llm_static.LLMStaticAsset; import tech.illuin.wombat.module.llm_static.LLMStaticModule; +import tech.illuin.wombat.persistence.backend.push.WorkerProperties; +import java.nio.file.Path; +import java.time.Duration; import java.util.List; +import java.util.Optional; import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -95,7 +105,7 @@ void invalidEnvironmentThrowsConstraintViolationException() private static MonitoredEnvironments load() { Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); - return new MonitorConfig().provideMonitoredEnvironments(new TemplateProperties(TEMPLATE), mapper(), validator); + return new MonitorConfig().provideMonitoredEnvironments(wombatProperties(TEMPLATE), mapper(), validator); } /** @@ -120,12 +130,60 @@ private static Asset byType(List assets, AssetType type) .orElseThrow(() -> new AssertionError("Template declares no " + type + " asset")); } - private record TemplateProperties(String environmentsFile) implements MonitorProperties + private static WombatProperties wombatProperties(String environmentsFile) { - @Override - public Period heartbeat() + return new WombatProperties() { - return Period.parse("1s"); - } + @Override public NodeMode mode() { return NodeMode.standalone; } + @Override public WorkerProperties worker() + { + return new WorkerProperties() { + @Override public Optional id() { return Optional.empty(); } + @Override public Optional aggregator() { return Optional.empty(); } + }; + } + + @Override public MonitorProperties monitor() + { + return new MonitorProperties() + { + @Override public Period heartbeat() { return Period.parse("1s"); } + @Override public String environmentsFile() { return environmentsFile; } + }; + } + + @Override public WombatProperties.EvaluatorProperties evaluator() + { + return new WombatProperties.EvaluatorProperties() + { + @Override public CachedAssetEvaluatorProperties cache() { return null; } + }; + } + + @Override public WombatModuleProperties module() + { + return new WombatModuleProperties() + { + @Override public WombatModuleProperties.KubernetesApiProperties kubernetesApi() { return () -> true; } + @Override public WombatModuleProperties.LlmPrometheusProperties llmPrometheus() { return () -> true; } + @Override public WombatModuleProperties.LlmAnthropicProperties llmAnthropic() { return () -> true; } + @Override public WombatModuleProperties.LlmVertexAIProperties llmVertexAi() { return () -> true; } + @Override public WombatModuleProperties.ExtensionProperties extension() { return () -> Path.of("extensions"); } + }; + } + + @Override + public WombatProperties.ConnectorProperties connector() + { + return new WombatProperties.ConnectorProperties() + { + @Override + public BoaviztaProperties boavizta() { return null; } + + @Override + public EcologitsProperties ecologits() { return null; } + }; + } + }; } } diff --git a/src/test/java/tech/illuin/wombat/engine/secret/database/crypto/RestoredSecretStartupTest.java b/src/test/java/tech/illuin/wombat/engine/secret/database/crypto/RestoredSecretStartupTest.java index c966cfb..efbda92 100644 --- a/src/test/java/tech/illuin/wombat/engine/secret/database/crypto/RestoredSecretStartupTest.java +++ b/src/test/java/tech/illuin/wombat/engine/secret/database/crypto/RestoredSecretStartupTest.java @@ -83,20 +83,21 @@ public Map start() } return Map.ofEntries( + Map.entry("persistence.backend.sqlite.jdbc-url", "jdbc:sqlite:" + this.target), Map.entry("quarkus.datasource.jdbc.url", "jdbc:sqlite:" + this.target), Map.entry("quarkus.flyway.clean-at-start", "false"), Map.entry("quarkus.flyway.migrate-at-start", "false"), Map.entry("quarkus.scheduler.enabled", "true"), - Map.entry("monitor.environments-file", "src/test/resources/monitored-restored-secret.yaml"), - Map.entry("backup.enabled", "true"), - Map.entry("backup.restore-on-startup", "true"), - Map.entry("backup.cleanup.enabled", "false"), - Map.entry("backup.s3.endpoint", this.localstack.getEndpoint().toString()), - Map.entry("backup.s3.bucket", "secret-restore-test"), - Map.entry("backup.s3.region", this.localstack.getRegion()), - Map.entry("backup.s3.key-prefix", "restored/"), - Map.entry("backup.s3.access-key", this.localstack.getAccessKey()), - Map.entry("backup.s3.secret-key", this.localstack.getSecretKey()), + Map.entry("wombat.monitor.environments-file", "src/test/resources/monitored-restored-secret.yaml"), + Map.entry("persistence.backend.sqlite.backup.enabled", "true"), + Map.entry("persistence.backend.sqlite.backup.restore-on-startup", "true"), + Map.entry("persistence.backend.sqlite.backup.cleanup.enabled", "false"), + Map.entry("persistence.backend.sqlite.backup.s3.endpoint", this.localstack.getEndpoint().toString()), + Map.entry("persistence.backend.sqlite.backup.s3.bucket", "secret-restore-test"), + Map.entry("persistence.backend.sqlite.backup.s3.region", this.localstack.getRegion()), + Map.entry("persistence.backend.sqlite.backup.s3.key-prefix", "restored/"), + Map.entry("persistence.backend.sqlite.backup.s3.access-key", this.localstack.getAccessKey()), + Map.entry("persistence.backend.sqlite.backup.s3.secret-key", this.localstack.getSecretKey()), Map.entry("wombat.secret.write-mode", "DATABASE"), Map.entry("wombat.secret.database.active-kek-id", "test"), Map.entry("wombat.secret.database.kek.test", TEST_KEK) diff --git a/src/test/java/tech/illuin/wombat/engine/secret/database/persistence/PostgresDatabaseSecretStoreTest.java b/src/test/java/tech/illuin/wombat/engine/secret/database/persistence/PostgresDatabaseSecretStoreTest.java index fb87d13..19cf946 100644 --- a/src/test/java/tech/illuin/wombat/engine/secret/database/persistence/PostgresDatabaseSecretStoreTest.java +++ b/src/test/java/tech/illuin/wombat/engine/secret/database/persistence/PostgresDatabaseSecretStoreTest.java @@ -158,6 +158,10 @@ public Map getConfigOverrides() "01234567890123456789012345678901".getBytes(StandardCharsets.UTF_8) ); return Map.ofEntries( + Map.entry("persistence.backend.mode", "postgresql"), + Map.entry("persistence.backend.postgresql.jdbc-url", jdbcUrl), + Map.entry("persistence.backend.postgresql.username", PostgresTestContainer.username()), + Map.entry("persistence.backend.postgresql.password", PostgresTestContainer.password()), Map.entry("quarkus.datasource.db-kind", "postgresql"), Map.entry("quarkus.datasource.username", PostgresTestContainer.username()), Map.entry("quarkus.datasource.password", PostgresTestContainer.password()), diff --git a/src/test/java/tech/illuin/wombat/persistence/BackendVerificationTest.java b/src/test/java/tech/illuin/wombat/persistence/BackendVerificationTest.java new file mode 100644 index 0000000..99d9304 --- /dev/null +++ b/src/test/java/tech/illuin/wombat/persistence/BackendVerificationTest.java @@ -0,0 +1,186 @@ +package tech.illuin.wombat.persistence; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import org.junit.jupiter.api.Test; +import tech.illuin.wombat.persistence.backend.PersistenceEngine; +import tech.illuin.wombat.persistence.dialect.JsonPathDialect; + +import java.sql.SQLException; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** Verifies that the SQLite file backend is mounted when configured with a file-based JDBC URL. */ +@QuarkusTest +@TestProfile(SQLiteFileBackendVerificationTest.SQLiteFileProfile.class) +class SQLiteFileBackendVerificationTest +{ + @Inject PersistenceEngine engine; + @Inject PersistenceBackendProperties backendProps; + @Inject JsonPathDialect jsonPathDialect; + + @Test + void sqlite_file_backend_is_mounted() + { + assertTrue(this.engine.isMounted("sqlite"), "SQLite backend should be mounted"); + assertEquals(BackendMode.sqlite, this.backendProps.mode(), "Backend mode should be sqlite"); + assertSame(JsonPathDialect.SQLITE, this.jsonPathDialect, "JsonPathDialect should be SQLITE"); + } + + public static class SQLiteFileProfile implements QuarkusTestProfile + { + @Override + public Map getConfigOverrides() + { + return Map.of( + "persistence.backend.mode", "sqlite", + "persistence.backend.sqlite.jdbc-url", "jdbc:sqlite:target/wombat-verify-file.db" + ); + } + } +} + +/** Verifies that the SQLite in-memory backend is mounted when configured with jdbc:sqlite::memory:. */ +@QuarkusTest +@TestProfile(SQLiteInMemoryBackendVerificationTest.SQLiteInMemoryProfile.class) +class SQLiteInMemoryBackendVerificationTest +{ + @Inject PersistenceEngine engine; + @Inject PersistenceBackendProperties backendProps; + @Inject JsonPathDialect jsonPathDialect; + + @Test + void sqlite_in_memory_backend_is_mounted() + { + assertTrue(this.engine.isMounted("sqlite"), "SQLite backend should be mounted"); + assertEquals(BackendMode.sqlite, this.backendProps.mode(), "Backend mode should be sqlite"); + assertSame(JsonPathDialect.SQLITE, this.jsonPathDialect, "JsonPathDialect should be SQLITE"); + } + + public static class SQLiteInMemoryProfile implements QuarkusTestProfile + { + @Override + public Map getConfigOverrides() + { + return Map.of( + "persistence.backend.mode", "sqlite", + "persistence.backend.sqlite.jdbc-url", "jdbc:sqlite::memory:" + ); + } + } +} + +/** Verifies that the PostgreSQL backend is mounted when configured with mode=postgresql. */ +@QuarkusTest +@TestProfile(PostgresBackendVerificationTest.PostgresProfile.class) +class PostgresBackendVerificationTest +{ + @Inject PersistenceEngine engine; + @Inject PersistenceBackendProperties backendProps; + @Inject JsonPathDialect jsonPathDialect; + + @Test + void postgres_backend_is_mounted() + { + assertTrue(this.engine.isMounted("postgresql"), "PostgreSQL backend should be mounted"); + assertEquals(BackendMode.postgresql, this.backendProps.mode(), "Backend mode should be postgresql"); + assertSame(JsonPathDialect.POSTGRESQL, this.jsonPathDialect, "JsonPathDialect should be POSTGRESQL"); + } + + public static class PostgresProfile implements QuarkusTestProfile + { + @Override + public Map getConfigOverrides() + { + String jdbcUrl; + try { + jdbcUrl = tech.illuin.wombat.persistence.backend.postgres.PostgresTestContainer.freshDatabase("wombat_backend_verify"); + } + catch (SQLException e) { + throw new IllegalStateException("Could not create PostgreSQL test database", e); + } + return Map.ofEntries( + Map.entry("persistence.backend.mode", "postgresql"), + Map.entry("persistence.backend.postgresql.jdbc-url", jdbcUrl), + Map.entry("persistence.backend.postgresql.username", tech.illuin.wombat.persistence.backend.postgres.PostgresTestContainer.username()), + Map.entry("persistence.backend.postgresql.password", tech.illuin.wombat.persistence.backend.postgres.PostgresTestContainer.password()), + Map.entry("quarkus.datasource.db-kind", "postgresql"), + Map.entry("quarkus.datasource.username", tech.illuin.wombat.persistence.backend.postgres.PostgresTestContainer.username()), + Map.entry("quarkus.datasource.password", tech.illuin.wombat.persistence.backend.postgres.PostgresTestContainer.password()), + Map.entry("quarkus.datasource.jdbc.url", jdbcUrl), + Map.entry("quarkus.flyway.locations", "classpath:db/migration/postgresql"), + Map.entry("quarkus.flyway.migrate-at-start", "true"), + Map.entry("quarkus.flyway.clean-at-start", "true"), + Map.entry("quarkus.flyway.clean-disabled", "false") + ); + } + } +} + +/** Verifies that aggregator mode keeps the configured backend (not overridden). */ +@QuarkusTest +@TestProfile(AggregatorBackendVerificationTest.AggregatorProfile.class) +class AggregatorBackendVerificationTest +{ + @Inject PersistenceEngine engine; + @Inject PersistenceBackendProperties backendProps; + @Inject JsonPathDialect jsonPathDialect; + + @Test + void aggregator_keeps_configured_backend() + { + assertTrue(this.engine.isMounted("sqlite"), "Aggregator should mount sqlite backend"); + assertEquals(BackendMode.sqlite, this.backendProps.mode(), "Aggregator mode should keep sqlite"); + assertSame(JsonPathDialect.SQLITE, this.jsonPathDialect, "Aggregator should use SQLITE dialect"); + } + + public static class AggregatorProfile implements QuarkusTestProfile + { + @Override + public Map getConfigOverrides() + { + return Map.of( + "wombat.mode", "aggregator", + "persistence.backend.mode", "sqlite", + "persistence.backend.sqlite.jdbc-url", "jdbc:sqlite::memory:" + ); + } + } +} + +/** Verifies that worker mode forces sqlite in-memory via NodeModeConfigSourceFactory. */ +@QuarkusTest +@TestProfile(WorkerBackendVerificationTest.WorkerProfile.class) +class WorkerBackendVerificationTest +{ + @Inject PersistenceEngine engine; + @Inject PersistenceBackendProperties backendProps; + @Inject JsonPathDialect jsonPathDialect; + + @Test + void worker_forces_sqlite_in_memory_override() + { + assertTrue(this.engine.isMounted("sqlite"), "Worker should mount sqlite backend (override)"); + assertEquals(BackendMode.sqlite, this.backendProps.mode(), "Worker mode should override to sqlite"); + assertSame(JsonPathDialect.SQLITE, this.jsonPathDialect, "Worker should use SQLITE dialect"); + } + + public static class WorkerProfile implements QuarkusTestProfile + { + @Override + public Map getConfigOverrides() + { + return Map.of( + "wombat.mode", "worker", + "wombat.worker.id", "test-worker", + "wombat.worker.aggregator.url", "http://localhost:0", + "wombat.worker.aggregator.api-key", "unused", + "persistence.backend.mode", "sqlite", + "persistence.backend.sqlite.jdbc-url", "jdbc:sqlite::memory:" + ); + } + } +} diff --git a/src/test/resources/application.yaml b/src/test/resources/application.yaml index a4716c1..bd056dc 100644 --- a/src/test/resources/application.yaml +++ b/src/test/resources/application.yaml @@ -4,19 +4,16 @@ quarkus: uri: "http://localhost:0/boavizta" ecologits-api-url: uri: "http://localhost:0/ecologits" + # Database configuration is handled dynamically by PersistenceBackendConfigSourceFactory + # based on persistence.backend.mode. The values below are build-time defaults only + # and are overridden at runtime with priority Integer.MAX_VALUE - 1000. datasource: db-kind: sqlite - jdbc: - url: "jdbc:sqlite:target/wombat-test.db" - min-size: 0 hibernate-orm: schema-management: strategy: none flyway: - migrate-at-start: true - clean-at-start: true - clean-disabled: false - locations: classpath:db/migration/${quarkus.datasource.db-kind} + locations: classpath:db/migration/sqlite scheduler: enabled: false devservices: @@ -35,20 +32,48 @@ quarkus: reuse-data-file: true report: false -monitor: - heartbeat: "1s" - environments-file: monitored-environments-test.yaml +persistence: + backend: + mode: sqlite + sqlite: + jdbc-url: "jdbc:sqlite:target/wombat-test.db" + backup: + enabled: false + s3: + endpoint: "http://localhost:0" + bucket: "wombat-test" + region: "us-east-1" + key-prefix: "tests/" + access-key: "test" + secret-key: "test" -module: - extension: - path: "/target/extensions" + compaction: + cron: "0 1 * * * ?" + window: 1h + delay: 1m -backup: - enabled: false - s3: - endpoint: "http://localhost:0" - bucket: "wombat-test" - region: "us-east-1" - key-prefix: "tests/" - access-key: "test" - secret-key: "test" +ui: + max-date-range: + duration: 1 + unit: YEARS + +wombat: + monitor: + heartbeat: "1s" + environments-file: monitored-environments-test.yaml + + evaluator: + cache: + enabled: true + maximum-size: 5000 + expire-after-write: 10m + + module: + kubernetes-api: + enabled: true + llm-prometheus: + enabled: true + llm-anthropic: + enabled: true + extension: + path: "/target/extensions"