From 23176ffe607475a0ee100350533c9d5609c90282 Mon Sep 17 00:00:00 2001 From: Mate Czagany Date: Tue, 17 Nov 2026 10:20:22 +0100 Subject: [PATCH 1/5] fix: Kafka subtask reported idle on slow split assignment --- README.md | 4 + .../kafka/AssignmentRecordingSource.java | 30 ++ .../kafka/InitialAssignmentDoneEvent.java | 35 ++ .../InitialAssignmentSignalingEnumerator.java | 255 +++++++++++++ ...KafkaAdminIdleAdvanceReadinessChecker.java | 23 ++ .../connector/kafka/KafkaReadPositions.java | 16 + ...KafkaRecordTimestampWatermarkStrategy.java | 92 ++++- .../kafka/AssignmentRecordingSourceTest.java | 45 +++ ...tialAssignmentSignalingEnumeratorTest.java | 339 ++++++++++++++++++ ...aAdminIdleAdvanceReadinessCheckerTest.java | 40 +++ .../kafka/KafkaReadPositionsTest.java | 14 + ...aRecordTimestampWatermarkStrategyTest.java | 180 +++++++++- .../flinkrunner/KafkaSourceWatermarkIT.java | 47 +++ ...urce_watermark_idle_advance_unassigned.sql | 46 +++ 14 files changed, 1161 insertions(+), 5 deletions(-) create mode 100644 connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentDoneEvent.java create mode 100644 connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java create mode 100644 connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumeratorTest.java create mode 100644 flink-sql-runner/src/test/resources/sql/kafka_source_watermark_idle_advance_unassigned.sql diff --git a/README.md b/README.md index 79c284f1..3177171f 100644 --- a/README.md +++ b/README.md @@ -346,6 +346,10 @@ After a restore, a partition can take longer than the timeout to produce its fir Such a partition has been read to the end when its split started at the end of its log, or when its consumer has since reached that end, past any commit markers, and every record it returned has been read. On a fresh start from the earliest or the committed offsets, the connector first looks up where each split starts. A partition's first row moves its watermark to just below that row at once, without waiting for `scan.source-watermark.min-records` records, so downstream operators see the source subtask as active, and hold their watermark below the row, before any watermark that would pass it. +A source subtask is reported idle only while none of its partitions has produced a row and each has been read to the end of its log, as far as the latest successful log-end fetch shows. +It is reported active again when a partition produces its first row, or before that when a log-end fetch shows unread records or a new partition is assigned. +A subtask is not reported idle before the enumerator has told it that its first partition discovery is done and assigned, since until then it may still be about to get a partition with a backlog, which downstream operators would move their watermark past meanwhile: its first on a fresh start, or after a restore one that was added to the topic while the job was down. +Until then it holds back the watermark of downstream operators, for as long as that first discovery takes. The log ends of all of a source subtask's partitions are fetched together, at most once per broker-check TTL, which therefore also bounds how far an idle partition's watermark trails the wall clock: keep it close to the idle-advance timeout. The safety margin must cover clock skew between the Kafka brokers and the TaskManagers, and the time a record takes to become readable after the broker timestamps it. diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSource.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSource.java index e08474ee..718274c0 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSource.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSource.java @@ -18,8 +18,11 @@ import java.util.List; import org.apache.flink.annotation.Internal; import org.apache.flink.api.connector.source.Source; +import org.apache.flink.api.connector.source.SourceEvent; import org.apache.flink.api.connector.source.SourceReader; import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit; /** @@ -28,6 +31,10 @@ * advancement needs these to tell whether the splits that have not produced a row have anything * left to read. Flink assigns splits on the source task's thread, whose positions the generators * read. + * + *

Also records when the enumerator has finished the reader's first assignment, which its + * enumerator, wrapped in an {@link InitialAssignmentSignalingEnumerator}, tells every reader: a + * reader is idle only from then on. */ @Internal public final class AssignmentRecordingSource @@ -37,6 +44,20 @@ public AssignmentRecordingSource(Source delega super(delegate); } + @Override + public SplitEnumerator createEnumerator( + SplitEnumeratorContext enumContext) throws Exception { + return new InitialAssignmentSignalingEnumerator<>(enumContext, delegate::createEnumerator); + } + + @Override + public SplitEnumerator restoreEnumerator( + SplitEnumeratorContext enumContext, EnumChkT checkpoint) + throws Exception { + return new InitialAssignmentSignalingEnumerator<>( + enumContext, context -> delegate.restoreEnumerator(context, checkpoint)); + } + @Override public SourceReader createReader(SourceReaderContext readerContext) throws Exception { @@ -57,5 +78,14 @@ public void addSplits(List splits) { KafkaReadPositions.assigned(split.getTopicPartition(), split.getStartingOffset())); super.addSplits(splits); } + + @Override + public void handleSourceEvents(SourceEvent sourceEvent) { + if (sourceEvent instanceof InitialAssignmentDoneEvent) { + KafkaReadPositions.initialAssignmentDone(); + } else { + super.handleSourceEvents(sourceEvent); + } + } } } diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentDoneEvent.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentDoneEvent.java new file mode 100644 index 00000000..520d420b --- /dev/null +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentDoneEvent.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2026 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.flinkrunner.connector.kafka; + +import java.io.Serial; +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.connector.source.SourceEvent; + +/** + * Tells a reader that the enumerator has finished its first partition discovery and has assigned + * the reader every split that discovery had for it, which may be none. + */ +@Internal +public final class InitialAssignmentDoneEvent implements SourceEvent { + + @Serial private static final long serialVersionUID = 1L; + + @Override + public String toString() { + return "InitialAssignmentDoneEvent"; + } +} diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java new file mode 100644 index 00000000..03c9e72f --- /dev/null +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java @@ -0,0 +1,255 @@ +/* + * Copyright © 2026 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.flinkrunner.connector.kafka; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; +import javax.annotation.Nullable; +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.connector.source.ReaderInfo; +import org.apache.flink.api.connector.source.SourceEvent; +import org.apache.flink.api.connector.source.SourceSplit; +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.api.connector.source.SplitsAssignment; +import org.apache.flink.metrics.groups.SplitEnumeratorMetricGroup; +import org.apache.flink.util.function.FunctionWithException; + +/** + * An enumerator that sends each reader an {@link InitialAssignmentDoneEvent} once the enumerator it + * wraps has assigned the splits of its first discovery. + * + *

Flink tells a reader which splits it gets, but not that it gets none, so a reader without + * splits cannot tell whether the enumerator is still about to assign it one. The enumerator has + * finished its first assignment once every async call it made from {@code start()}, and every one + * the handlers of those calls made, has been handled: a Kafka enumerator lists the partitions in + * one call, and looks up where their splits start and assigns them in the handler of a second. + * Readers registered by then get the event at that point, and readers that register later get it + * after the splits that were pending for them. + * + *

If a handler of those calls fails, no reader gets the event; such a failure fails the job. + */ +@Internal +public final class InitialAssignmentSignalingEnumerator + implements SplitEnumerator { + + private final SplitEnumeratorContext context; + private final SplitEnumerator delegate; + + /** The async calls of the first assignment whose handler has not returned yet. */ + private int pendingInitialCalls; + + /** Whether the coordinator thread is in {@code start()} or in a handler of such a call. */ + private boolean inInitialAssignment; + + private boolean started; + private boolean initialAssignmentDone; + + /** + * @param enumeratorFactory creates or restores the wrapped enumerator with the context it is + * given, through which this enumerator sees its async calls + */ + public InitialAssignmentSignalingEnumerator( + SplitEnumeratorContext context, + FunctionWithException< + SplitEnumeratorContext, SplitEnumerator, Exception> + enumeratorFactory) + throws Exception { + this.context = checkNotNull(context, "Enumerator context must not be null."); + this.delegate = + checkNotNull( + enumeratorFactory.apply(new CallTrackingContext()), + "Delegate enumerator must not be null."); + } + + @Override + public void start() { + inInitialAssignment = true; + try { + delegate.start(); + } finally { + inInitialAssignment = false; + } + + started = true; + signalIfInitialAssignmentDone(); + } + + @Override + public void addReader(int subtaskId) { + // Assigns the reader the splits that were pending for it, ahead of the event. + delegate.addReader(subtaskId); + if (initialAssignmentDone) { + context.sendEventToSourceReader(subtaskId, new InitialAssignmentDoneEvent()); + } + } + + private void signalIfInitialAssignmentDone() { + if (initialAssignmentDone || !started || pendingInitialCalls > 0) { + return; + } + + initialAssignmentDone = true; + for (int subtaskId : context.registeredReaders().keySet()) { + context.sendEventToSourceReader(subtaskId, new InitialAssignmentDoneEvent()); + } + } + + @Override + public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { + delegate.handleSplitRequest(subtaskId, requesterHostname); + } + + @Override + public void addSplitsBack(List splits, int subtaskId) { + delegate.addSplitsBack(splits, subtaskId); + } + + @Override + public EnumChkT snapshotState(long checkpointId) throws Exception { + return delegate.snapshotState(checkpointId); + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + delegate.notifyCheckpointComplete(checkpointId); + } + + @Override + public void notifyCheckpointAborted(long checkpointId) throws Exception { + delegate.notifyCheckpointAborted(checkpointId); + } + + @Override + public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { + delegate.handleSourceEvent(subtaskId, sourceEvent); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + /** + * The context the wrapped enumerator works with: the coordinator's, except that it counts the + * async calls of the first assignment. + */ + private final class CallTrackingContext implements SplitEnumeratorContext { + + @Override + public void callAsync(Callable callable, BiConsumer handler) { + context.callAsync(callable, tracked(handler)); + } + + @Override + public void callAsync( + Callable callable, + BiConsumer handler, + long initialDelayMillis, + long periodMillis) { + // Only the first run of a periodic call belongs to the first assignment. + context.callAsync(callable, tracked(handler), initialDelayMillis, periodMillis); + } + + /** Counts {@code handler}'s first run, if the call is made during the first assignment. */ + private BiConsumer tracked(BiConsumer handler) { + if (!inInitialAssignment) { + return handler; + } + + pendingInitialCalls++; + var ran = new AtomicBoolean(); + return (result, error) -> { + if (ran.getAndSet(true)) { + handler.accept(result, error); + return; + } + + inInitialAssignment = true; + try { + handler.accept(result, error); + } finally { + inInitialAssignment = false; + } + + // Not reached if the handler throws: the call then stays pending. + pendingInitialCalls--; + signalIfInitialAssignmentDone(); + }; + } + + @Override + public SplitEnumeratorMetricGroup metricGroup() { + return context.metricGroup(); + } + + @Override + public void sendEventToSourceReader(int subtaskId, SourceEvent event) { + context.sendEventToSourceReader(subtaskId, event); + } + + @Override + public void sendEventToSourceReader(int subtaskId, int attemptNumber, SourceEvent event) { + context.sendEventToSourceReader(subtaskId, attemptNumber, event); + } + + @Override + public int currentParallelism() { + return context.currentParallelism(); + } + + @Override + public Map registeredReaders() { + return context.registeredReaders(); + } + + @Override + public Map> registeredReadersOfAttempts() { + return context.registeredReadersOfAttempts(); + } + + @Override + public void assignSplits(SplitsAssignment newSplitAssignments) { + context.assignSplits(newSplitAssignments); + } + + @Override + public void assignSplit(SplitT split, int subtask) { + context.assignSplit(split, subtask); + } + + @Override + public void signalNoMoreSplits(int subtask) { + context.signalNoMoreSplits(subtask); + } + + @Override + public void runInCoordinatorThread(Runnable runnable) { + context.runInCoordinatorThread(runnable); + } + + @Override + public void setIsProcessingBacklog(boolean isProcessingBacklog) { + context.setIsProcessingBacklog(isProcessingBacklog); + } + } +} diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java index 567ade28..b01a9ca7 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java @@ -136,6 +136,29 @@ && assignmentsWithoutRows(state).stream() .allMatch(position -> isReadToEnd(state, position)); } + /** + * Whether no partition assigned to this subtask has produced a row and each has been read to the + * end of its log, as the latest fetch that succeeded saw it. A subtask may be reported idle only + * then: one that is reported idle with records left lets downstream operators move their + * watermark past them. False for a partition assigned since that fetch, whose log end is unknown. + * + *

A failed broker check does not change the answer, as it says nothing about unread records. + * An idle subtask that turned active on one would hold back the watermark of a join or union + * until the brokers answer again. + */ + public boolean assignmentsHaveNothingToRead(long currentTimeMillis) { + var assignments = KafkaReadPositions.assignments(); + var producedRow = brokerState().partitions; + // A partition that has produced a row settles it without the brokers, which a subtask busy + // reading would otherwise ask on its task thread every broker-check TTL. + if (assignments.stream().anyMatch(position -> producedRow.contains(position.partition()))) { + return false; + } + + var state = refreshIfDue(currentTimeMillis); + return assignments.stream().allMatch(position -> isReadToEnd(state, position)); + } + /** Whether {@code position} has read to the end of its log, as the latest fetch saw it. */ private boolean isReadToEnd(BrokerState state, ReadPosition position) { var logEnd = state.logEnds.get(position.partition()); diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositions.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositions.java index d304594b..d09d43be 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositions.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositions.java @@ -61,6 +61,9 @@ public final class KafkaReadPositions { /** The position of the partition this thread consumed a record from last. */ @Nullable private ReadPosition latest; + /** Whether the enumerator has said that the reader's first assignment is done. */ + private boolean initialAssignmentDone; + private KafkaReadPositions() {} /** Records that the record at {@code offset} has been consumed, before it is deserialized. */ @@ -84,6 +87,19 @@ public static void assigned(TopicPartition partition, long startingOffset) { startingOffset; } + /** + * Records that the enumerator has assigned the reader on this thread every split its first + * discovery had for it ({@link InitialAssignmentDoneEvent}). + */ + public static void initialAssignmentDone() { + FOR_THREAD.get().initialAssignmentDone = true; + } + + /** Whether the enumerator has finished the first assignment of the reader on this thread. */ + static boolean isInitialAssignmentDone() { + return FOR_THREAD.get().initialAssignmentDone; + } + /** The positions of the partitions assigned to the reader on this thread. */ static List assignments() { return FOR_THREAD.get().positions.values().stream() diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java index c4e6b9e5..92139c42 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java @@ -48,6 +48,13 @@ public final class KafkaRecordTimestampWatermarkStrategy implements WatermarkStr private final MillisClock clock; @Nullable private final KafkaAdminIdleAdvanceReadinessChecker idleAdvanceReadinessChecker; + /** + * Whether the generator of the reader's main output has been created. Flink deserializes one + * strategy per source subtask and creates that generator from it first, along with the main + * output; those of the splits follow as they are assigned. + */ + private transient boolean mainOutputGeneratorCreated; + /** Takes an idle advance readiness checker if and only if idle advancement is enabled. */ public KafkaRecordTimestampWatermarkStrategy( WatermarkEmitStrategy emitStrategy, @@ -80,12 +87,16 @@ public KafkaRecordTimestampWatermarkStrategy( public WatermarkGenerator createWatermarkGenerator( WatermarkGeneratorSupplier.Context context) { + boolean forMainOutput = !mainOutputGeneratorCreated; + mainOutputGeneratorCreated = true; + return new AdaptiveKafkaRecordTimestampWatermarkGenerator( emitStrategy, sourceWatermarkConfig, clock, context.getInputActivityClock(), - idleAdvanceReadinessChecker); + idleAdvanceReadinessChecker, + forMainOutput); } /** @@ -109,6 +120,10 @@ public WatermarkGenerator createWatermarkGenerator( * which also moves for records that produce no row. Offsets that hold no record, such as * transaction markers, count as read once the consumer has moved past them ({@link * KafkaFetchPositions}) and the split has read every record before them. + * + *

The generator of the reader's main output sees no row, as Kafka emits each through its + * split's output. It reports the subtask idle and active instead: see {@link + * #reportSubtaskIdleness}. */ private static final class AdaptiveKafkaRecordTimestampWatermarkGenerator implements WatermarkGenerator { @@ -132,6 +147,9 @@ private static final class AdaptiveKafkaRecordTimestampWatermarkGenerator /** Measures idle time; stops while the split is paused or backpressured. */ private final RelativeClock inputActivityClock; + /** Whether this is the generator of the reader's main output, not of a split. */ + private final boolean forMainOutput; + /** Highest Kafka record timestamp observed by this generator. */ private long maxTimestamp = Long.MIN_VALUE; @@ -156,6 +174,9 @@ private static final class AdaptiveKafkaRecordTimestampWatermarkGenerator /** Whether this output was marked idle before receiving any records. */ private boolean markedIdle; + /** Whether the main output was marked active, which keeps the subtask from being idle. */ + private boolean markedActive; + /** * Number of records observed, including the first record that cannot produce a lateness sample. */ @@ -169,12 +190,14 @@ private AdaptiveKafkaRecordTimestampWatermarkGenerator( SourceWatermarkConfig config, MillisClock clock, RelativeClock inputActivityClock, - @Nullable KafkaAdminIdleAdvanceReadinessChecker idleAdvanceReadinessChecker) { + @Nullable KafkaAdminIdleAdvanceReadinessChecker idleAdvanceReadinessChecker, + boolean forMainOutput) { this.emitStrategy = emitStrategy; this.config = config; this.clock = clock; this.inputActivityClock = inputActivityClock; this.idleAdvanceReadinessChecker = idleAdvanceReadinessChecker; + this.forMainOutput = forMainOutput; this.idleSinceInputActivityMillis = inputActivityClock.relativeTimeMillis(); } @@ -251,6 +274,11 @@ private void emitIdleWatermarkIfReady(WatermarkOutput output) { // counts as idle from the first check after its latest progress, up to one interval late, // never early. long inputActivityMillis = inputActivityClock.relativeTimeMillis(); + if (forMainOutput && recordCount == 0) { + reportSubtaskIdleness(output, inputActivityMillis); + return; + } + long progress = readPosition != null ? readPosition.nextOffset() : recordCount; if (progress != progressAtLastIdleCheck) { progressAtLastIdleCheck = progress; @@ -298,6 +326,66 @@ private void emitIdleWatermarkIfReady(WatermarkOutput output) { frontierMillis - outOfOrdernessMillis - config.idleAdvanceSafetyMarginMillis() - 1); } + /** + * Reports the subtask idle or active through the reader's main output. + * + *

Flink reports the subtask idle when the main output is marked idle before the splits have + * made the subtask active. In Flink 2.3.0 that takes a watermark from every split that is not + * idle, so a row from each; from FLINK-40504 on, a row from one of them. So a main output + * marked idle by its own timeout would leave a subtask idle that has a backlog, for as long as + * those rows take, and downstream operators would move their watermark past that backlog. + * + *

The main output is therefore idle only while the subtask has nothing to read, and active + * otherwise, which also turns a subtask active again that was reported idle before it was + * assigned a partition the enumerator discovered later. It is also active until the enumerator + * has said that the subtask's first assignment is done ({@link InitialAssignmentDoneEvent}). + * Until then the subtask may still be about to get a partition with a backlog: its first on a + * fresh start, or after a restore one that was added to the topic while the job was down. Not + * marking the main output idle would not be enough for a restored subtask: Flink counts a main + * output that has said nothing as idle, and would report the subtask idle once its splits, read + * to the end, are. + */ + private void reportSubtaskIdleness(WatermarkOutput output, long inputActivityMillis) { + long currentTimeMillis = clock.currentTimeMillis(); + int assignments = KafkaReadPositions.assignments().size(); + + if (!KafkaReadPositions.isInitialAssignmentDone()) { + markMainOutputActive(output); + return; + } + + if (assignments > 0 + && !idleAdvanceReadinessChecker.assignmentsHaveNothingToRead(currentTimeMillis)) { + idleSinceInputActivityMillis = inputActivityMillis; + markMainOutputActive(output); + return; + } + + if (assignments != progressAtLastIdleCheck) { + progressAtLastIdleCheck = assignments; + idleSinceInputActivityMillis = inputActivityMillis; + } + + if (inputActivityMillis - idleSinceInputActivityMillis < config.idleAdvanceTimeoutMillis()) { + return; + } + + if (!markedIdle && idleAdvanceReadinessChecker.isReady(currentTimeMillis)) { + output.markIdle(); + markedIdle = true; + markedActive = false; + } + } + + /** Marks the main output active, which keeps Flink from reporting the subtask idle. */ + private void markMainOutputActive(WatermarkOutput output) { + if (!markedActive) { + output.markActive(); + markedActive = true; + markedIdle = false; + } + } + /** Emits {@code watermark} if it moves the watermark forward, as Flink requires. */ private void emitIfHigher(WatermarkOutput output, long watermark) { if (watermark > lastEmittedWatermark) { diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSourceTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSourceTest.java index fd72bfc3..4c883473 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSourceTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSourceTest.java @@ -19,8 +19,10 @@ import static org.assertj.core.api.Assertions.tuple; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -28,8 +30,11 @@ import java.util.ArrayList; import java.util.List; import org.apache.flink.api.connector.source.Source; +import org.apache.flink.api.connector.source.SourceEvent; import org.apache.flink.api.connector.source.SourceReader; import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit; import org.apache.kafka.common.TopicPartition; import org.junit.jupiter.api.AfterEach; @@ -69,6 +74,46 @@ void testRecordsAssignedSplitsBeforeTheReaderGetsThem() throws Exception { .containsExactly(tuple(PARTITION, 42L))); } + @Test + void testRecordsInitialAssignmentDoneEventWithoutPassingItOn() throws Exception { + // KafkaSourceReader does not know the event. + var recordingReader = recordingReader(); + + assertThat(KafkaReadPositions.isInitialAssignmentDone()).isFalse(); + + recordingReader.handleSourceEvents(new InitialAssignmentDoneEvent()); + + assertThat(KafkaReadPositions.isInitialAssignmentDone()).isTrue(); + verify(reader, never()).handleSourceEvents(any()); + } + + @Test + void testPassesOnOtherSourceEvents() throws Exception { + SourceEvent event = mock(SourceEvent.class); + + recordingReader().handleSourceEvents(event); + + verify(reader).handleSourceEvents(event); + assertThat(KafkaReadPositions.isInitialAssignmentDone()).isFalse(); + } + + @Test + @SuppressWarnings("unchecked") + void testWrapsCreatedAndRestoredEnumeratorToSignalInitialAssignment() throws Exception { + Source source = mock(Source.class); + SplitEnumerator enumerator = mock(SplitEnumerator.class); + when(source.createEnumerator(any())).thenReturn(enumerator); + when(source.restoreEnumerator(any(), any())).thenReturn(enumerator); + SplitEnumeratorContext context = mock(SplitEnumeratorContext.class); + var recordingSource = new AssignmentRecordingSource<>(source); + + assertThat(recordingSource.createEnumerator(context)) + .isInstanceOf(InitialAssignmentSignalingEnumerator.class); + assertThat(recordingSource.restoreEnumerator(context, "checkpoint")) + .isInstanceOf(InitialAssignmentSignalingEnumerator.class); + verify(source).restoreEnumerator(any(), eq("checkpoint")); + } + @Test void testForwardsReaderCallbacks() throws Exception { // KafkaSourceReader commits offsets on checkpoint completion; alignment pauses splits. diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumeratorTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumeratorTest.java new file mode 100644 index 00000000..1bfa06ea --- /dev/null +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumeratorTest.java @@ -0,0 +1,339 @@ +/* + * Copyright © 2026 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.flinkrunner.connector.kafka; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.function.BiConsumer; +import org.apache.flink.api.common.serialization.SimpleStringSchema; +import org.apache.flink.api.connector.source.ReaderInfo; +import org.apache.flink.api.connector.source.SourceEvent; +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.api.connector.source.SplitsAssignment; +import org.apache.flink.connector.kafka.source.KafkaSource; +import org.apache.flink.connector.kafka.source.enumerator.KafkaSourceEnumState; +import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer; +import org.apache.flink.connector.kafka.source.enumerator.subscriber.KafkaSubscriber; +import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit; +import org.apache.flink.metrics.groups.SplitEnumeratorMetricGroup; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.common.TopicPartition; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Tests {@link InitialAssignmentSignalingEnumerator} around the Kafka connector's own enumerator, + * whose async calls it follows. The partitions come from a subscriber, so no broker is needed. + */ +class InitialAssignmentSignalingEnumeratorTest { + + private static final int PARALLELISM = 3; + private static final TopicPartition PARTITION_0 = new TopicPartition("topic", 0); + private static final TopicPartition PARTITION_1 = new TopicPartition("topic", 1); + + private final FixedSubscriber subscriber = new FixedSubscriber(PARTITION_0, PARTITION_1); + private final Coordinator coordinator = new Coordinator(); + private SplitEnumerator enumerator; + + @AfterEach + void closeEnumerator() throws Exception { + if (enumerator != null) { + enumerator.close(); + } + } + + @Test + void testSignalsReadersOnceSplitsOfFirstDiscoveryAreAssigned() throws Exception { + enumerator = createEnumerator(false); + registerReaders(); + enumerator.start(); + + assertThat(coordinator.sent).isEmpty(); + + // Lists the partitions, and asks where their splits start in a second call. + coordinator.runNextAsyncCall(); + + assertThat(coordinator.sent).isEmpty(); + + coordinator.runNextAsyncCall(); + + // Two partitions for three readers: one gets no split, and still the event. + assertThat(coordinator.sent).filteredOn(sent -> sent.startsWith("splits")).hasSize(2); + assertThat(coordinator.sent.subList(2, 5)) + .containsExactly("done to 0", "done to 1", "done to 2"); + assertThat(coordinator.asyncCalls).isEmpty(); + } + + @Test + void testSignalsReaderRegisteringLaterAfterItsPendingSplits() throws Exception { + enumerator = createEnumerator(false); + enumerator.start(); + coordinator.runAsyncCalls(); + + assertThat(coordinator.sent).isEmpty(); + + registerReaders(); + + assertThat(coordinator.sent).filteredOn(sent -> sent.startsWith("done")).hasSize(3); + for (int reader = 0; reader < PARALLELISM; reader++) { + int splits = coordinator.sent.indexOf("splits to " + reader); + assertThat(splits).isLessThan(coordinator.sent.indexOf("done to " + reader)); + } + } + + @Test + void testSignalsRestoredReadersWhenDiscoveryFindsNothingNew() throws Exception { + // The restored readers hold every partition, so the enumerator makes no second call. + enumerator = restoreEnumerator(true); + registerReaders(); + enumerator.start(); + + assertThat(coordinator.sent).isEmpty(); + + coordinator.runNextAsyncCall(); + + assertThat(coordinator.sent).containsExactly("done to 0", "done to 1", "done to 2"); + assertThat(coordinator.asyncCalls).isEmpty(); + } + + @Test + void testSignalsRestoredReadersWhenEnumeratorKeepsItsDiscoveryUnfinished() throws Exception { + // Restored from a checkpoint taken before its first discovery finished, the enumerator finds + // nothing new and leaves its own flag unset, which therefore cannot stand in for the event. + enumerator = restoreEnumerator(false); + registerReaders(); + enumerator.start(); + coordinator.runAsyncCalls(); + + assertThat(coordinator.sent).containsExactly("done to 0", "done to 1", "done to 2"); + assertThat(enumerator.snapshotState(1L).initialDiscoveryFinished()).isFalse(); + } + + @Test + void testSignalsOnceWithPeriodicDiscovery() throws Exception { + enumerator = createEnumerator(true); + registerReaders(); + enumerator.start(); + coordinator.runAsyncCalls(); + int sentByFirstDiscovery = coordinator.sent.size(); + + subscriber.partitions.add(new TopicPartition("topic", 2)); + coordinator.runPeriodicAsyncCalls(); + + assertThat(coordinator.sent.subList(sentByFirstDiscovery, coordinator.sent.size())) + .singleElement() + .asString() + .startsWith("splits to "); + } + + @Test + void testDoesNotSignalWhenDiscoveryFails() throws Exception { + enumerator = createEnumerator(false); + registerReaders(); + subscriber.failing = true; + enumerator.start(); + + assertThatThrownBy(coordinator::runAsyncCalls) + .hasMessageContaining("Failed to list subscribed topic partitions"); + assertThat(coordinator.sent).isEmpty(); + } + + @Test + @SuppressWarnings("unchecked") + void testForwardsEnumeratorCallbacks() throws Exception { + SplitEnumerator delegate = mock(SplitEnumerator.class); + var signaling = new InitialAssignmentSignalingEnumerator<>(coordinator, context -> delegate); + SourceEvent event = mock(SourceEvent.class); + var splits = List.of(new KafkaPartitionSplit(PARTITION_0, 5L)); + + signaling.handleSplitRequest(1, "host"); + signaling.addSplitsBack(splits, 1); + signaling.snapshotState(7L); + signaling.notifyCheckpointComplete(7L); + signaling.notifyCheckpointAborted(8L); + signaling.handleSourceEvent(1, event); + signaling.close(); + + verify(delegate).handleSplitRequest(1, "host"); + verify(delegate).addSplitsBack(splits, 1); + verify(delegate).snapshotState(7L); + verify(delegate).notifyCheckpointComplete(7L); + verify(delegate).notifyCheckpointAborted(8L); + verify(delegate).handleSourceEvent(1, event); + verify(delegate).close(); + } + + private SplitEnumerator createEnumerator( + boolean periodicDiscovery) throws Exception { + var source = kafkaSource(periodicDiscovery); + return new InitialAssignmentSignalingEnumerator<>(coordinator, source::createEnumerator); + } + + /** An enumerator restored with both partitions assigned. */ + private SplitEnumerator restoreEnumerator( + boolean initialDiscoveryFinished) throws Exception { + var source = kafkaSource(false); + var checkpoint = + new KafkaSourceEnumState( + List.of( + new KafkaPartitionSplit(PARTITION_0, 5L), new KafkaPartitionSplit(PARTITION_1, 7L)), + List.of(), + initialDiscoveryFinished); + return new InitialAssignmentSignalingEnumerator<>( + coordinator, context -> source.restoreEnumerator(context, checkpoint)); + } + + private KafkaSource kafkaSource(boolean periodicDiscovery) { + return KafkaSource.builder() + .setBootstrapServers("localhost:1") + .setKafkaSubscriber(subscriber) + .setStartingOffsets(OffsetsInitializer.earliest()) + .setValueOnlyDeserializer(new SimpleStringSchema()) + .setProperty("partition.discovery.interval.ms", periodicDiscovery ? "1000" : "-1") + .build(); + } + + private void registerReaders() { + for (int reader = 0; reader < PARALLELISM; reader++) { + coordinator.readers.put(reader, new ReaderInfo(reader, "host")); + enumerator.addReader(reader); + } + } + + /** The partitions of the topic, without asking a broker. */ + private static final class FixedSubscriber implements KafkaSubscriber { + + private final Set partitions = new HashSet<>(); + private boolean failing; + + private FixedSubscriber(TopicPartition... partitions) { + this.partitions.addAll(List.of(partitions)); + } + + @Override + public Set getSubscribedTopicPartitions(AdminClient adminClient) { + if (failing) { + throw new IllegalStateException("Broker cannot be reached"); + } + return new HashSet<>(partitions); + } + } + + /** + * The source coordinator's context: it runs the enumerator's async calls, each with its handler, + * when the test says so, and records what the enumerator sends to which reader. + */ + private static final class Coordinator implements SplitEnumeratorContext { + + private final Map readers = new TreeMap<>(); + private final List sent = new ArrayList<>(); + private final Deque asyncCalls = new ArrayDeque<>(); + private final List periodicAsyncCalls = new ArrayList<>(); + + private void runNextAsyncCall() { + asyncCalls.remove().run(); + } + + private void runAsyncCalls() { + while (!asyncCalls.isEmpty()) { + runNextAsyncCall(); + } + } + + /** The next run of every periodic call, and what its handler calls in turn. */ + private void runPeriodicAsyncCalls() { + asyncCalls.addAll(periodicAsyncCalls); + runAsyncCalls(); + } + + private Runnable asyncCall(Callable callable, BiConsumer handler) { + return () -> { + T result = null; + Throwable error = null; + try { + result = callable.call(); + } catch (Throwable t) { + error = t; + } + handler.accept(result, error); + }; + } + + @Override + public void callAsync(Callable callable, BiConsumer handler) { + asyncCalls.add(asyncCall(callable, handler)); + } + + @Override + public void callAsync( + Callable callable, + BiConsumer handler, + long initialDelayMillis, + long periodMillis) { + Runnable asyncCall = asyncCall(callable, handler); + asyncCalls.add(asyncCall); + periodicAsyncCalls.add(asyncCall); + } + + @Override + public void assignSplits(SplitsAssignment newSplitAssignments) { + new TreeMap<>(newSplitAssignments.assignment()) + .forEach((reader, splits) -> sent.add("splits to " + reader)); + } + + @Override + public void sendEventToSourceReader(int subtaskId, SourceEvent event) { + assertThat(event).isInstanceOf(InitialAssignmentDoneEvent.class); + sent.add("done to " + subtaskId); + } + + @Override + public Map registeredReaders() { + return readers; + } + + @Override + public int currentParallelism() { + return PARALLELISM; + } + + @Override + public SplitEnumeratorMetricGroup metricGroup() { + return null; + } + + @Override + public void signalNoMoreSplits(int subtask) {} + + @Override + public void runInCoordinatorThread(Runnable runnable) { + runnable.run(); + } + } +} diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java index c55f2e39..42ebb31c 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java @@ -277,6 +277,46 @@ void testPartitionProducingRowsDoesNotHoldBackSplitsWithoutRows() throws Excepti assertThat(checker.assignmentsWithoutRowsReadToEnd(1000L)).isTrue(); } + @Test + void testSubtaskWithoutAssignmentsHasNothingToRead() throws Exception { + var checker = newReadinessChecker(new FakeAdmin()); + + assertThat(checker.assignmentsHaveNothingToRead(1000L)).isTrue(); + } + + @Test + void testPartitionThatProducedRowKeepsSubtaskFromHavingNothingToRead() throws Exception { + // Its split advances the watermark itself once it is idle, so the subtask stays active. + var admin = new FakeAdmin(); + admin.highWatermark(PARTITION_0, 10L); + admin.highWatermark(PARTITION_1, 20L); + var checker = newReadinessChecker(admin); + KafkaReadPositions.assigned(PARTITION_0, 10L); + KafkaReadPositions.assigned(PARTITION_1, 20L); + + assertThat(checker.assignmentsHaveNothingToRead(1000L)).isTrue(); + + checker.register(PARTITION_0); + + assertThat(checker.assignmentsHaveNothingToRead(1000L)).isFalse(); + } + + @Test + void testPartitionAssignedSinceLatestFetchKeepsSubtaskFromHavingNothingToRead() throws Exception { + var admin = new FakeAdmin(); + admin.highWatermark(PARTITION_0, 10L); + admin.highWatermark(PARTITION_1, 20L); + var checker = newReadinessChecker(admin); + KafkaReadPositions.assigned(PARTITION_0, 10L); + + assertThat(checker.assignmentsHaveNothingToRead(1000L)).isTrue(); + + KafkaReadPositions.assigned(PARTITION_1, 20L); + + assertThat(checker.assignmentsHaveNothingToRead(1001L)).isFalse(); + assertThat(checker.assignmentsHaveNothingToRead(1000L + BROKER_CHECK_TTL_MILLIS)).isTrue(); + } + @Test void testSplitRestoredBeforeTrailingCommitMarkerIsReadToEndOnceConsumerMovesPastIt() throws Exception { diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositionsTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositionsTest.java index 8e850d86..fdc8fc9b 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositionsTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositionsTest.java @@ -51,6 +51,20 @@ void testRecordsAssignedPartitionsWithWhereTheirSplitsStart() { .allMatch(position -> position.nextOffset() == KafkaReadPositions.BEFORE_FIRST_RECORD); } + @Test + void testRecordsThatInitialAssignmentIsDoneUntilCleared() { + assertThat(KafkaReadPositions.isInitialAssignmentDone()).isFalse(); + + KafkaReadPositions.initialAssignmentDone(); + + assertThat(KafkaReadPositions.isInitialAssignmentDone()).isTrue(); + + // A restarted task runs on a new thread, which starts without it. + KafkaReadPositions.clear(); + + assertThat(KafkaReadPositions.isInitialAssignmentDone()).isFalse(); + } + @Test void testConsumingAssignedPartitionKeepsWhereItsSplitStarted() { KafkaReadPositions.assigned(new TopicPartition(topic, 0), 42L); diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java index dd925d92..4b619bb6 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java @@ -838,6 +838,13 @@ void testRowReactivatesIdleSubtaskBelowItBeforeWatermarkThatReleasesIt() { reader.newSplit(); reader.tick(6); + // Not before the enumerator's first discovery is done: it may still find a partition that + // was added while the job was down. + assertThat(reader.downstream).isEmpty(); + + reader.initialAssignmentDone(); + reader.tick(6); + assertThat(reader.downstream).containsExactly("IDLE"); final long timestamp = source.wallClockMillis(); @@ -870,6 +877,149 @@ void testSubtaskReadingRowsIsNotReportedIdleBeforeItsFirstIdleWatermark() { .contains("watermark@" + source.caughtUpWatermark()); } + @Test + void testSubtaskWithoutSplitsIsReportedIdleOnlyOnceItsFirstAssignmentIsDone() { + // The enumerator may still be about to assign the subtask a partition with a backlog, which + // downstream operators would move their watermark past while the subtask is reported idle. + final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); + final Reader reader = new Reader(source); + reader.tick(150); + + assertThat(reader.downstream).isEmpty(); + + reader.initialAssignmentDone(); + reader.tick(1); + + assertThat(reader.downstream).containsExactly("IDLE"); + } + + @Test + void testSubtaskAssignedItsSplitsAfterIdleAdvanceTimeoutIsNeverReportedIdle() { + // A fresh start whose enumerator takes longer than the idle-advance timeout to assign splits. + final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); + final Reader reader = new Reader(source); + reader.tick(6); + final ReaderSplit reading = reader.newSplit(); + final ReaderSplit backlog = reader.newSplit(); + backlog.log().append(10); + reader.initialAssignmentDone(); + + final long timestamp = source.wallClockMillis(); + reading.appendAndRead(timestamp); + reader.tick(10); + + assertThat(reader.downstream).containsExactly("record@" + timestamp); + } + + @Test + void testSubtaskReportedIdleBeforeItsSplitsWereAssignedTurnsActiveWhileOneAwaitsItsFirstRow() { + // The subtask got no partition at first, and the enumerator discovers two for it later. In + // Flink 2.3.0 the splits have no watermark between them until the backlog split produces its + // first row, so only the main output can tell Flink that the subtask is active again. + final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); + final Reader reader = new Reader(source); + reader.initialAssignmentDone(); + reader.tick(6); + + assertThat(reader.downstream).containsExactly("IDLE"); + + final ReaderSplit reading = reader.newSplit(); + final ReaderSplit backlog = reader.newSplit(); + backlog.log().append(10); + + final long timestamp = source.wallClockMillis(); + reading.appendAndRead(timestamp); + reader.tick(10); + + assertThat(reader.downstream).containsExactly("IDLE", "record@" + timestamp, "ACTIVE"); + } + + @Test + void testSubtaskReportedIdleBeforeItsSplitWasAssignedTurnsActiveBeforeBacklogProducesRow() { + // Until the first row nothing else tells Flink that the subtask is active, with or without + // FLINK-40504, and downstream operators move their watermark past the backlog meanwhile. + final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); + final Reader reader = new Reader(source); + reader.initialAssignmentDone(); + reader.tick(6); + final ReaderSplit backlog = reader.newSplit(); + backlog.log().append(10); + reader.tick(1); + + assertThat(reader.downstream).containsExactly("IDLE", "ACTIVE"); + + backlog.read(0L); + + assertThat(reader.downstream) + .containsExactly("IDLE", "ACTIVE", "record@0", "watermark@" + firstRowWatermark(0L)); + } + + @Test + void testSubtaskReadingBacklogIsNotReportedIdleWhileEmptySplitIsNotIdleYet() { + // The splits are assigned before the idle-advance timeout of the main output has passed, so + // that it passes before the empty split's own, until which the splits have no watermark. + final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); + final Reader reader = new Reader(source); + reader.tick(3); + + final ReaderSplit backlog = reader.newSplit(); + reader.newSplit(); + backlog.log().append(10); + for (long timestamp = 0; timestamp < 3; timestamp++) { + backlog.read(timestamp); + reader.tick(1); + } + reader.tick(10); + + assertThat(reader.downstream) + .containsExactly("record@0", "record@1", "record@2", "watermark@" + firstRowWatermark(0L)); + } + + @Test + void testSubtaskTurnedActiveIsReportedIdleAgainOnceNothingIsLeftToRead() { + // Every record of the split produces no row, so no watermark would end the subtask's wait. + final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); + final Reader reader = new Reader(source); + reader.initialAssignmentDone(); + reader.tick(6); + final ReaderSplit split = reader.newSplit(); + split.log().append(2); + reader.tick(1); + + assertThat(reader.downstream).containsExactly("IDLE", "ACTIVE"); + + split.log().consumeNext(); + split.log().consumeNext(); + reader.tick(4); + + assertThat(reader.downstream).containsExactly("IDLE", "ACTIVE"); + + reader.tick(2); + + assertThat(reader.downstream).containsExactly("IDLE", "ACTIVE", "IDLE"); + } + + @Test + void testFailedBrokerCheckDoesNotTurnIdleSubtaskActive() { + // A check that fails, as when the brokers answer too slowly, says nothing about unread + // records. Turned active on it, a quiet source would hold back the watermark of a join. + final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); + final Reader reader = new Reader(source); + reader.newSplit(); + reader.newSplit(); + reader.initialAssignmentDone(); + reader.tick(6); + + assertThat(reader.downstream).containsExactly("IDLE"); + + source.broker.logAppendTime = false; + reader.tick(5); + source.broker.logAppendTime = true; + reader.tick(10); + + assertThat(reader.downstream).containsExactly("IDLE"); + } + @Test void testRestoredSplitWithBacklogIsNotMarkedIdleBeforeItsFirstRow() { // After a restore, a split can take longer than the idle-advance timeout to produce its first @@ -999,6 +1149,9 @@ private static final class Source { /** The interceptor on the source's consumer, which publishes how far it has fetched. */ private final KafkaFetchPositions fetchPositions = new KafkaFetchPositions(); + @Nullable private KafkaRecordTimestampWatermarkStrategy strategy; + private boolean mainOutputGeneratorCreated; + private Source(SourceWatermarkConfig config, long wallClockMillis) { this(config, wallClockMillis, false); } @@ -1029,9 +1182,23 @@ private Split newSplit(WatermarkOutputMultiplexer multiplexer, String splitId) { this, multiplexer.getImmediateOutput(splitId), multiplexer.getDeferredOutput(splitId)); } + /** The subtask's strategy: Flink deserializes one, and creates every generator from it. */ private KafkaRecordTimestampWatermarkStrategy strategy() { - return new KafkaRecordTimestampWatermarkStrategy( - WatermarkEmitStrategy.ON_PERIODIC, config, this::wallClockMillis, broker); + if (strategy == null) { + strategy = + new KafkaRecordTimestampWatermarkStrategy( + WatermarkEmitStrategy.ON_PERIODIC, config, this::wallClockMillis, broker); + } + return strategy; + } + + /** A split's generator, which Flink creates after the one of the reader's main output. */ + private WatermarkGenerator newSplitGenerator(RelativeClock inputActivityClock) { + if (!mainOutputGeneratorCreated) { + strategy().createWatermarkGenerator(context(new PausableRelativeClock(monotonicClock))); + mainOutputGeneratorCreated = true; + } + return strategy().createWatermarkGenerator(context(inputActivityClock)); } private void advanceMillis(long millis) { @@ -1183,7 +1350,7 @@ private Split(Source source, WatermarkOutput eventOutput, WatermarkOutput period this.inputActivityClock = new PausableRelativeClock(source.monotonicClock); this.eventOutput = eventOutput; this.periodicOutput = periodicOutput; - this.generator = source.strategy().createWatermarkGenerator(context(inputActivityClock)); + this.generator = source.newSplitGenerator(inputActivityClock); } private void append(int records) { @@ -1254,6 +1421,8 @@ private static final class Reader { private final ReaderOutput output; private Reader(Source source) { + checkState(!source.mainOutputGeneratorCreated, "The reader creates the main output first"); + source.mainOutputGeneratorCreated = true; this.source = source; this.timestampsAndWatermarks = new ProgressiveTimestampsAndWatermarks<>( @@ -1270,6 +1439,11 @@ private Reader(Source source) { timestampsAndWatermarks.createMainOutput(new RecordingOutput(), new NoOpListener()); } + /** The enumerator's event that the reader's first assignment is done, as its reader records. */ + private void initialAssignmentDone() { + KafkaReadPositions.initialAssignmentDone(); + } + /** A split assigned at the start of its partition's log. */ private ReaderSplit newSplit() { return newSplit(0L); diff --git a/flink-sql-runner/src/test/java/com/datasqrl/flinkrunner/KafkaSourceWatermarkIT.java b/flink-sql-runner/src/test/java/com/datasqrl/flinkrunner/KafkaSourceWatermarkIT.java index deb3ca69..dfee1114 100644 --- a/flink-sql-runner/src/test/java/com/datasqrl/flinkrunner/KafkaSourceWatermarkIT.java +++ b/flink-sql-runner/src/test/java/com/datasqrl/flinkrunner/KafkaSourceWatermarkIT.java @@ -44,6 +44,8 @@ class KafkaSourceWatermarkIT extends AbstractITSupport { private static final String TOPIC = "source-watermark-it"; private static final String IDLE_ADVANCE_TOPIC = "source-watermark-idle-advance-it"; private static final String NO_IDLE_ADVANCE_TOPIC = "source-watermark-no-idle-advance-it"; + private static final String IDLE_ADVANCE_UNASSIGNED_TOPIC = + "source-watermark-idle-advance-unassigned-it"; private static final String IDLE_ADVANCE_BACKLOG_TOPIC = "source-watermark-idle-advance-backlog-it"; private static final String IDLE_ADVANCE_REPLAY_TOPIC = "source-watermark-idle-advance-replay-it"; @@ -139,6 +141,37 @@ void givenSparseKafkaSourceWatermark_whenIdleAdvanceEnabled_thenWatermarkClosesW }); } + /** + * With a parallelism of two and one partition, one source subtask is assigned no split. It holds + * the watermark back until the enumerator tells it that its first assignment is done, and is + * reported idle from then on, so the other subtask's watermark closes the window. + */ + @Test + void givenSourceSubtaskWithoutPartition_whenIdleAdvanceEnabled_thenWatermarkClosesWindows() + throws Exception { + var resultDao = emptyResults(); + createTopic(IDLE_ADVANCE_UNASSIGNED_TOPIC, true); + // The cluster starts with one slot, and the job needs one per source subtask. + startTaskManager(); + + try { + runJob( + "/it/sqlfile/kafka_source_watermark_idle_advance_unassigned.sql", + jobId -> { + produceJsonRecord( + IDLE_ADVANCE_UNASSIGNED_TOPIC, "{\"id\":0,\"payload\":\"idle-record-0\"}"); + + untilAssert( + () -> { + assertThat(resultDao.getRowCount()).isEqualTo(1); + assertThat(resultDao.getTotalCount()).isEqualTo(1); + }); + }); + } finally { + stopTaskManager(); + } + } + /** * A fresh job reading history: one partition's newest record is already old when it is read, and * the partition is then idle. As the minimum across partitions it must not hold the source back @@ -366,6 +399,20 @@ private AlignmentResultDao emptyAlignmentResults() { return resultDao; } + /** Adds a TaskManager, and with it a slot, to the cluster. */ + private void startTaskManager() throws Exception { + Container.ExecResult result = flinkContainer.execInContainer("bin/taskmanager.sh", "start"); + + assertThat(result.getExitCode()) + .withFailMessage(result.getStdout() + result.getStderr()) + .isZero(); + } + + /** Stops the TaskManager that was started last. */ + private void stopTaskManager() throws Exception { + flinkContainer.execInContainer("bin/taskmanager.sh", "stop"); + } + private String readFlinkLogs() throws Exception { Container.ExecResult result = flinkContainer.execInContainer( diff --git a/flink-sql-runner/src/test/resources/sql/kafka_source_watermark_idle_advance_unassigned.sql b/flink-sql-runner/src/test/resources/sql/kafka_source_watermark_idle_advance_unassigned.sql new file mode 100644 index 00000000..8e17a4c9 --- /dev/null +++ b/flink-sql-runner/src/test/resources/sql/kafka_source_watermark_idle_advance_unassigned.sql @@ -0,0 +1,46 @@ +-- Two source subtasks for a topic with one partition: one of them is assigned no split, and has to +-- be reported idle for the watermark to close windows. +SET 'parallelism.default' = '2'; + +CREATE TABLE idle_advance_unassigned_events ( + id BIGINT, + payload STRING, + ts TIMESTAMP_LTZ(3) METADATA FROM 'timestamp', + WATERMARK FOR ts AS SOURCE_WATERMARK() +) WITH ( + 'connector' = 'kafka-safe', + 'topic' = 'source-watermark-idle-advance-unassigned-it', + 'properties.bootstrap.servers' = 'redpanda:9092', + 'properties.group.id' = 'source-watermark-idle-advance-unassigned-it', + 'scan.startup.mode' = 'earliest-offset', + 'scan.source-watermark.min-records' = '250', + 'scan.source-watermark.min-out-of-orderness' = '50 ms', + 'scan.source-watermark.idle-advance-timeout' = '1 s', + 'scan.source-watermark.idle-advance-safety-margin' = '1 s', + 'scan.source-watermark.idle-advance-broker-check-timeout' = '1 s', + 'scan.source-watermark.idle-advance-broker-check-ttl' = '1 s', + 'format' = 'json' +); + +CREATE TABLE source_watermark_results ( + window_start TIMESTAMP(3) NOT NULL, + window_end TIMESTAMP(3) NOT NULL, + cnt BIGINT NOT NULL +) WITH ( + 'connector' = 'jdbc', + 'driver' = 'org.postgresql.Driver', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}', + 'password' = '${POSTGRES_PASSWORD}', + 'table-name' = 'source_watermark_results' +); + +INSERT INTO source_watermark_results +SELECT + CAST(window_start AS TIMESTAMP(3)), + CAST(window_end AS TIMESTAMP(3)), + COUNT(*) +FROM TABLE( + TUMBLE(TABLE idle_advance_unassigned_events, DESCRIPTOR(ts), INTERVAL '1' SECOND) +) +GROUP BY window_start, window_end; From a499539f0375faefcd71b07fb236fea66f05c60b Mon Sep 17 00:00:00 2001 From: Mate Czagany Date: Tue, 17 Nov 2026 15:15:53 +0100 Subject: [PATCH 2/5] Address PR comments --- README.md | 4 +- .../connector/kafka/ForwardingEnumerator.java | 84 +++++++ .../kafka/ForwardingEnumeratorContext.java | 113 ++++++++++ .../InitialAssignmentSignalingEnumerator.java | 111 +-------- ...KafkaAdminIdleAdvanceReadinessChecker.java | 18 +- ...KafkaRecordTimestampWatermarkStrategy.java | 213 +++++++++++------- ...aAdminIdleAdvanceReadinessCheckerTest.java | 18 ++ ...aRecordTimestampWatermarkStrategyTest.java | 68 +++++- 8 files changed, 435 insertions(+), 194 deletions(-) create mode 100644 connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/ForwardingEnumerator.java create mode 100644 connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/ForwardingEnumeratorContext.java diff --git a/README.md b/README.md index 3177171f..0e126063 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,9 @@ Such a partition has been read to the end when its split started at the end of i On a fresh start from the earliest or the committed offsets, the connector first looks up where each split starts. A partition's first row moves its watermark to just below that row at once, without waiting for `scan.source-watermark.min-records` records, so downstream operators see the source subtask as active, and hold their watermark below the row, before any watermark that would pass it. A source subtask is reported idle only while none of its partitions has produced a row and each has been read to the end of its log, as far as the latest successful log-end fetch shows. -It is reported active again when a partition produces its first row, or before that when a log-end fetch shows unread records or a new partition is assigned. +It is reported active again when a partition produces its first row, or before that when a new partition is assigned; while it is idle, it does not ask the brokers. +Turning active again holds back the watermark of a downstream operator only if no other source subtask that feeds the operator has moved it yet, since Flink leaves a subtask that turns active out of the operator's watermark until the subtask's own watermark has caught up with it. +So with a source parallelism above one, the backlog of a partition that is discovered while the job runs can still arrive late. A subtask is not reported idle before the enumerator has told it that its first partition discovery is done and assigned, since until then it may still be about to get a partition with a backlog, which downstream operators would move their watermark past meanwhile: its first on a fresh start, or after a restore one that was added to the topic while the job was down. Until then it holds back the watermark of downstream operators, for as long as that first discovery takes. diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/ForwardingEnumerator.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/ForwardingEnumerator.java new file mode 100644 index 00000000..d68689fd --- /dev/null +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/ForwardingEnumerator.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2026 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.flinkrunner.connector.kafka; + +import java.io.IOException; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.connector.source.SourceEvent; +import org.apache.flink.api.connector.source.SourceSplit; +import org.apache.flink.api.connector.source.SplitEnumerator; + +/** + * An enumerator that forwards every call to the enumerator it wraps, so that a wrapper overrides + * only what it changes. + * + *

The wrapped enumerator comes from {@link #delegate()}, not from the constructor as in {@link + * ForwardingSource}: a wrapper may have to create it with a context of its own, which it can only + * build once it is constructed itself. + */ +@Internal +public abstract class ForwardingEnumerator + implements SplitEnumerator { + + protected abstract SplitEnumerator delegate(); + + @Override + public void start() { + delegate().start(); + } + + @Override + public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { + delegate().handleSplitRequest(subtaskId, requesterHostname); + } + + @Override + public void addSplitsBack(List splits, int subtaskId) { + delegate().addSplitsBack(splits, subtaskId); + } + + @Override + public void addReader(int subtaskId) { + delegate().addReader(subtaskId); + } + + @Override + public EnumChkT snapshotState(long checkpointId) throws Exception { + return delegate().snapshotState(checkpointId); + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + delegate().notifyCheckpointComplete(checkpointId); + } + + @Override + public void notifyCheckpointAborted(long checkpointId) throws Exception { + delegate().notifyCheckpointAborted(checkpointId); + } + + @Override + public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { + delegate().handleSourceEvent(subtaskId, sourceEvent); + } + + @Override + public void close() throws IOException { + delegate().close(); + } +} diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/ForwardingEnumeratorContext.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/ForwardingEnumeratorContext.java new file mode 100644 index 00000000..89464fc6 --- /dev/null +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/ForwardingEnumeratorContext.java @@ -0,0 +1,113 @@ +/* + * Copyright © 2026 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.flinkrunner.connector.kafka; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.function.BiConsumer; +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.connector.source.ReaderInfo; +import org.apache.flink.api.connector.source.SourceEvent; +import org.apache.flink.api.connector.source.SourceSplit; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.api.connector.source.SplitsAssignment; +import org.apache.flink.metrics.groups.SplitEnumeratorMetricGroup; + +/** + * An enumerator context that forwards every call to the context it wraps, so that a wrapper + * overrides only what it changes. + */ +@Internal +public abstract class ForwardingEnumeratorContext + implements SplitEnumeratorContext { + + protected final SplitEnumeratorContext delegate; + + protected ForwardingEnumeratorContext(SplitEnumeratorContext delegate) { + this.delegate = checkNotNull(delegate, "Delegate enumerator context must not be null."); + } + + @Override + public SplitEnumeratorMetricGroup metricGroup() { + return delegate.metricGroup(); + } + + @Override + public void sendEventToSourceReader(int subtaskId, SourceEvent event) { + delegate.sendEventToSourceReader(subtaskId, event); + } + + @Override + public void sendEventToSourceReader(int subtaskId, int attemptNumber, SourceEvent event) { + delegate.sendEventToSourceReader(subtaskId, attemptNumber, event); + } + + @Override + public int currentParallelism() { + return delegate.currentParallelism(); + } + + @Override + public Map registeredReaders() { + return delegate.registeredReaders(); + } + + @Override + public Map> registeredReadersOfAttempts() { + return delegate.registeredReadersOfAttempts(); + } + + @Override + public void assignSplits(SplitsAssignment newSplitAssignments) { + delegate.assignSplits(newSplitAssignments); + } + + @Override + public void assignSplit(SplitT split, int subtask) { + delegate.assignSplit(split, subtask); + } + + @Override + public void signalNoMoreSplits(int subtask) { + delegate.signalNoMoreSplits(subtask); + } + + @Override + public void callAsync(Callable callable, BiConsumer handler) { + delegate.callAsync(callable, handler); + } + + @Override + public void callAsync( + Callable callable, + BiConsumer handler, + long initialDelayMillis, + long periodMillis) { + delegate.callAsync(callable, handler, initialDelayMillis, periodMillis); + } + + @Override + public void runInCoordinatorThread(Runnable runnable) { + delegate.runInCoordinatorThread(runnable); + } + + @Override + public void setIsProcessingBacklog(boolean isProcessingBacklog) { + delegate.setIsProcessingBacklog(isProcessingBacklog); + } +} diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java index 03c9e72f..06083668 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java @@ -17,21 +17,13 @@ import static org.apache.flink.util.Preconditions.checkNotNull; -import java.io.IOException; -import java.util.List; -import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; -import javax.annotation.Nullable; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.connector.source.ReaderInfo; -import org.apache.flink.api.connector.source.SourceEvent; import org.apache.flink.api.connector.source.SourceSplit; import org.apache.flink.api.connector.source.SplitEnumerator; import org.apache.flink.api.connector.source.SplitEnumeratorContext; -import org.apache.flink.api.connector.source.SplitsAssignment; -import org.apache.flink.metrics.groups.SplitEnumeratorMetricGroup; import org.apache.flink.util.function.FunctionWithException; /** @@ -50,7 +42,7 @@ */ @Internal public final class InitialAssignmentSignalingEnumerator - implements SplitEnumerator { + extends ForwardingEnumerator { private final SplitEnumeratorContext context; private final SplitEnumerator delegate; @@ -81,6 +73,11 @@ public InitialAssignmentSignalingEnumerator( "Delegate enumerator must not be null."); } + @Override + protected SplitEnumerator delegate() { + return delegate; + } + @Override public void start() { inInitialAssignment = true; @@ -114,46 +111,15 @@ private void signalIfInitialAssignmentDone() { } } - @Override - public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { - delegate.handleSplitRequest(subtaskId, requesterHostname); - } - - @Override - public void addSplitsBack(List splits, int subtaskId) { - delegate.addSplitsBack(splits, subtaskId); - } - - @Override - public EnumChkT snapshotState(long checkpointId) throws Exception { - return delegate.snapshotState(checkpointId); - } - - @Override - public void notifyCheckpointComplete(long checkpointId) throws Exception { - delegate.notifyCheckpointComplete(checkpointId); - } - - @Override - public void notifyCheckpointAborted(long checkpointId) throws Exception { - delegate.notifyCheckpointAborted(checkpointId); - } - - @Override - public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { - delegate.handleSourceEvent(subtaskId, sourceEvent); - } - - @Override - public void close() throws IOException { - delegate.close(); - } - /** * The context the wrapped enumerator works with: the coordinator's, except that it counts the * async calls of the first assignment. */ - private final class CallTrackingContext implements SplitEnumeratorContext { + private final class CallTrackingContext extends ForwardingEnumeratorContext { + + private CallTrackingContext() { + super(context); + } @Override public void callAsync(Callable callable, BiConsumer handler) { @@ -196,60 +162,5 @@ private BiConsumer tracked(BiConsumer handler) { signalIfInitialAssignmentDone(); }; } - - @Override - public SplitEnumeratorMetricGroup metricGroup() { - return context.metricGroup(); - } - - @Override - public void sendEventToSourceReader(int subtaskId, SourceEvent event) { - context.sendEventToSourceReader(subtaskId, event); - } - - @Override - public void sendEventToSourceReader(int subtaskId, int attemptNumber, SourceEvent event) { - context.sendEventToSourceReader(subtaskId, attemptNumber, event); - } - - @Override - public int currentParallelism() { - return context.currentParallelism(); - } - - @Override - public Map registeredReaders() { - return context.registeredReaders(); - } - - @Override - public Map> registeredReadersOfAttempts() { - return context.registeredReadersOfAttempts(); - } - - @Override - public void assignSplits(SplitsAssignment newSplitAssignments) { - context.assignSplits(newSplitAssignments); - } - - @Override - public void assignSplit(SplitT split, int subtask) { - context.assignSplit(split, subtask); - } - - @Override - public void signalNoMoreSplits(int subtask) { - context.signalNoMoreSplits(subtask); - } - - @Override - public void runInCoordinatorThread(Runnable runnable) { - context.runInCoordinatorThread(runnable); - } - - @Override - public void setIsProcessingBacklog(boolean isProcessingBacklog) { - context.setIsProcessingBacklog(isProcessingBacklog); - } } } diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java index b01a9ca7..9883ca88 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java @@ -31,6 +31,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -143,10 +144,21 @@ && assignmentsWithoutRows(state).stream() * watermark past them. False for a partition assigned since that fetch, whose log end is unknown. * *

A failed broker check does not change the answer, as it says nothing about unread records. - * An idle subtask that turned active on one would hold back the watermark of a join or union - * until the brokers answer again. */ public boolean assignmentsHaveNothingToRead(long currentTimeMillis) { + return assignmentsHaveNothingToRead(() -> refreshIfDue(currentTimeMillis)); + } + + /** + * {@link #assignmentsHaveNothingToRead(long)} as of the latest fetch, without asking the brokers. + * A subtask that is reported idle asks this, so that it leaves the brokers alone until it has + * something to read again. + */ + public boolean assignmentsHadNothingToReadAtLatestFetch() { + return assignmentsHaveNothingToRead(this::brokerState); + } + + private boolean assignmentsHaveNothingToRead(Supplier latestFetch) { var assignments = KafkaReadPositions.assignments(); var producedRow = brokerState().partitions; // A partition that has produced a row settles it without the brokers, which a subtask busy @@ -155,7 +167,7 @@ public boolean assignmentsHaveNothingToRead(long currentTimeMillis) { return false; } - var state = refreshIfDue(currentTimeMillis); + var state = latestFetch.get(); return assignments.stream().allMatch(position -> isReadToEnd(state, position)); } diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java index 92139c42..40823ba1 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java @@ -51,7 +51,8 @@ public final class KafkaRecordTimestampWatermarkStrategy implements WatermarkStr /** * Whether the generator of the reader's main output has been created. Flink deserializes one * strategy per source subtask and creates that generator from it first, along with the main - * output; those of the splits follow as they are assigned. + * output; those of the splits follow as they are assigned. With idle advancement enabled, the + * main output gets a {@link SubtaskIdlenessGenerator}. */ private transient boolean mainOutputGeneratorCreated; @@ -90,13 +91,137 @@ public WatermarkGenerator createWatermarkGenerator( boolean forMainOutput = !mainOutputGeneratorCreated; mainOutputGeneratorCreated = true; + if (forMainOutput && sourceWatermarkConfig.idleAdvanceTimeoutMillis() > 0) { + return new SubtaskIdlenessGenerator( + sourceWatermarkConfig.idleAdvanceTimeoutMillis(), + clock, + context.getInputActivityClock(), + idleAdvanceReadinessChecker); + } + return new AdaptiveKafkaRecordTimestampWatermarkGenerator( emitStrategy, sourceWatermarkConfig, clock, context.getInputActivityClock(), - idleAdvanceReadinessChecker, - forMainOutput); + idleAdvanceReadinessChecker); + } + + /** + * The generator of the reader's main output with idle advancement enabled. It sees no row, as + * Kafka emits each through its split's output, and reports the subtask idle or active instead. + * + *

Flink reports the subtask idle when the main output is marked idle before the splits have + * made the subtask active. In Flink 2.3.0 that takes a watermark from every split that is not + * idle, so a row from each; from FLINK-40504 on, a row from one of them. So a main output marked + * idle by its own timeout would leave a subtask idle that has a backlog, for as long as those + * rows take, and downstream operators would move their watermark past that backlog. + * + *

The main output is therefore idle only while the subtask has nothing to read, and active + * otherwise. It is also active until the enumerator has said that the subtask's first assignment + * is done ({@link InitialAssignmentDoneEvent}). Until then the subtask may still be about to get + * a partition with a backlog: its first on a fresh start, or after a restore one that was added + * to the topic while the job was down. The splits wait for that event too before they go idle: + * Flink checks them before the main output, and counts a main output that has not reported yet as + * idle. + * + *

A subtask reported idle turns active again when it is assigned a partition the enumerator + * discovered later. That holds back the watermark of a downstream operator only if no other + * subtask that feeds the operator has moved it yet: Flink leaves a subtask that turns active out + * of the operator's watermark until the subtask's own watermark has caught up with it. So where + * other subtasks feed the same operator, as with a keyed join and a source parallelism above one, + * a backlog in that partition still arrives late. + */ + private static final class SubtaskIdlenessGenerator implements WatermarkGenerator { + + /** What the main output has told Flink last. */ + private enum Reported { + NOTHING, + ACTIVE, + IDLE + } + + private final long idleAdvanceTimeoutMillis; + private final MillisClock clock; + private final KafkaAdminIdleAdvanceReadinessChecker idleAdvanceReadinessChecker; + + /** Measures idle time; stops while the task is backpressured. */ + private final RelativeClock inputActivityClock; + + /** Number of assigned partitions at the previous check that found nothing to read. */ + private int assignmentsAtLastIdleCheck; + + /** Input activity time since which the subtask has had nothing to read in its partitions. */ + private long idleSinceInputActivityMillis; + + private Reported reported = Reported.NOTHING; + + private SubtaskIdlenessGenerator( + long idleAdvanceTimeoutMillis, + MillisClock clock, + RelativeClock inputActivityClock, + KafkaAdminIdleAdvanceReadinessChecker idleAdvanceReadinessChecker) { + this.idleAdvanceTimeoutMillis = idleAdvanceTimeoutMillis; + this.clock = clock; + this.inputActivityClock = inputActivityClock; + this.idleAdvanceReadinessChecker = idleAdvanceReadinessChecker; + this.idleSinceInputActivityMillis = inputActivityClock.relativeTimeMillis(); + } + + @Override + public void onEvent(RowData event, long eventTimestamp, WatermarkOutput output) { + // Had Flink created a split's generator first, that split would get no watermark from here. + throw new IllegalStateException( + "The source reader's main output got a row, but Kafka emits rows through split outputs."); + } + + @Override + public void onPeriodicEmit(WatermarkOutput output) { + if (!KafkaReadPositions.isInitialAssignmentDone()) { + markActive(output); + return; + } + + long inputActivityMillis = inputActivityClock.relativeTimeMillis(); + long currentTimeMillis = clock.currentTimeMillis(); + int assignments = KafkaReadPositions.assignments().size(); + + if (assignments > 0 && !assignmentsHaveNothingToRead(currentTimeMillis)) { + idleSinceInputActivityMillis = inputActivityMillis; + markActive(output); + return; + } + + if (assignments != assignmentsAtLastIdleCheck) { + assignmentsAtLastIdleCheck = assignments; + idleSinceInputActivityMillis = inputActivityMillis; + } + + if (inputActivityMillis - idleSinceInputActivityMillis < idleAdvanceTimeoutMillis) { + return; + } + + if (reported != Reported.IDLE && idleAdvanceReadinessChecker.isReady(currentTimeMillis)) { + output.markIdle(); + reported = Reported.IDLE; + } + } + + private boolean assignmentsHaveNothingToRead(long currentTimeMillis) { + // A subtask reported idle does not ask the brokers again. What turns it active needs no new + // fetch: a partition assigned since the latest one, or a partition's first row. + return reported == Reported.IDLE + ? idleAdvanceReadinessChecker.assignmentsHadNothingToReadAtLatestFetch() + : idleAdvanceReadinessChecker.assignmentsHaveNothingToRead(currentTimeMillis); + } + + /** Marks the main output active, which keeps Flink from reporting the subtask idle. */ + private void markActive(WatermarkOutput output) { + if (reported != Reported.ACTIVE) { + output.markActive(); + reported = Reported.ACTIVE; + } + } } /** @@ -120,10 +245,6 @@ public WatermarkGenerator createWatermarkGenerator( * which also moves for records that produce no row. Offsets that hold no record, such as * transaction markers, count as read once the consumer has moved past them ({@link * KafkaFetchPositions}) and the split has read every record before them. - * - *

The generator of the reader's main output sees no row, as Kafka emits each through its - * split's output. It reports the subtask idle and active instead: see {@link - * #reportSubtaskIdleness}. */ private static final class AdaptiveKafkaRecordTimestampWatermarkGenerator implements WatermarkGenerator { @@ -147,9 +268,6 @@ private static final class AdaptiveKafkaRecordTimestampWatermarkGenerator /** Measures idle time; stops while the split is paused or backpressured. */ private final RelativeClock inputActivityClock; - /** Whether this is the generator of the reader's main output, not of a split. */ - private final boolean forMainOutput; - /** Highest Kafka record timestamp observed by this generator. */ private long maxTimestamp = Long.MIN_VALUE; @@ -174,9 +292,6 @@ private static final class AdaptiveKafkaRecordTimestampWatermarkGenerator /** Whether this output was marked idle before receiving any records. */ private boolean markedIdle; - /** Whether the main output was marked active, which keeps the subtask from being idle. */ - private boolean markedActive; - /** * Number of records observed, including the first record that cannot produce a lateness sample. */ @@ -190,14 +305,12 @@ private AdaptiveKafkaRecordTimestampWatermarkGenerator( SourceWatermarkConfig config, MillisClock clock, RelativeClock inputActivityClock, - @Nullable KafkaAdminIdleAdvanceReadinessChecker idleAdvanceReadinessChecker, - boolean forMainOutput) { + @Nullable KafkaAdminIdleAdvanceReadinessChecker idleAdvanceReadinessChecker) { this.emitStrategy = emitStrategy; this.config = config; this.clock = clock; this.inputActivityClock = inputActivityClock; this.idleAdvanceReadinessChecker = idleAdvanceReadinessChecker; - this.forMainOutput = forMainOutput; this.idleSinceInputActivityMillis = inputActivityClock.relativeTimeMillis(); } @@ -274,11 +387,6 @@ private void emitIdleWatermarkIfReady(WatermarkOutput output) { // counts as idle from the first check after its latest progress, up to one interval late, // never early. long inputActivityMillis = inputActivityClock.relativeTimeMillis(); - if (forMainOutput && recordCount == 0) { - reportSubtaskIdleness(output, inputActivityMillis); - return; - } - long progress = readPosition != null ? readPosition.nextOffset() : recordCount; if (progress != progressAtLastIdleCheck) { progressAtLastIdleCheck = progress; @@ -291,7 +399,10 @@ private void emitIdleWatermarkIfReady(WatermarkOutput output) { long currentTimeMillis = clock.currentTimeMillis(); if (recordCount == 0) { + // Not before the subtask's first assignment is done. Until then the main output keeps the + // subtask active, but only from its first check on, which Flink runs after the splits'. if (!markedIdle + && KafkaReadPositions.isInitialAssignmentDone() && idleAdvanceReadinessChecker.assignmentsWithoutRowsReadToEnd(currentTimeMillis)) { output.markIdle(); markedIdle = true; @@ -326,66 +437,6 @@ private void emitIdleWatermarkIfReady(WatermarkOutput output) { frontierMillis - outOfOrdernessMillis - config.idleAdvanceSafetyMarginMillis() - 1); } - /** - * Reports the subtask idle or active through the reader's main output. - * - *

Flink reports the subtask idle when the main output is marked idle before the splits have - * made the subtask active. In Flink 2.3.0 that takes a watermark from every split that is not - * idle, so a row from each; from FLINK-40504 on, a row from one of them. So a main output - * marked idle by its own timeout would leave a subtask idle that has a backlog, for as long as - * those rows take, and downstream operators would move their watermark past that backlog. - * - *

The main output is therefore idle only while the subtask has nothing to read, and active - * otherwise, which also turns a subtask active again that was reported idle before it was - * assigned a partition the enumerator discovered later. It is also active until the enumerator - * has said that the subtask's first assignment is done ({@link InitialAssignmentDoneEvent}). - * Until then the subtask may still be about to get a partition with a backlog: its first on a - * fresh start, or after a restore one that was added to the topic while the job was down. Not - * marking the main output idle would not be enough for a restored subtask: Flink counts a main - * output that has said nothing as idle, and would report the subtask idle once its splits, read - * to the end, are. - */ - private void reportSubtaskIdleness(WatermarkOutput output, long inputActivityMillis) { - long currentTimeMillis = clock.currentTimeMillis(); - int assignments = KafkaReadPositions.assignments().size(); - - if (!KafkaReadPositions.isInitialAssignmentDone()) { - markMainOutputActive(output); - return; - } - - if (assignments > 0 - && !idleAdvanceReadinessChecker.assignmentsHaveNothingToRead(currentTimeMillis)) { - idleSinceInputActivityMillis = inputActivityMillis; - markMainOutputActive(output); - return; - } - - if (assignments != progressAtLastIdleCheck) { - progressAtLastIdleCheck = assignments; - idleSinceInputActivityMillis = inputActivityMillis; - } - - if (inputActivityMillis - idleSinceInputActivityMillis < config.idleAdvanceTimeoutMillis()) { - return; - } - - if (!markedIdle && idleAdvanceReadinessChecker.isReady(currentTimeMillis)) { - output.markIdle(); - markedIdle = true; - markedActive = false; - } - } - - /** Marks the main output active, which keeps Flink from reporting the subtask idle. */ - private void markMainOutputActive(WatermarkOutput output) { - if (!markedActive) { - output.markActive(); - markedActive = true; - markedIdle = false; - } - } - /** Emits {@code watermark} if it moves the watermark forward, as Flink requires. */ private void emitIfHigher(WatermarkOutput output, long watermark) { if (watermark > lastEmittedWatermark) { diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java index 42ebb31c..cfc05d42 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java @@ -317,6 +317,24 @@ void testPartitionAssignedSinceLatestFetchKeepsSubtaskFromHavingNothingToRead() assertThat(checker.assignmentsHaveNothingToRead(1000L + BROKER_CHECK_TTL_MILLIS)).isTrue(); } + @Test + void testLatestFetchTellsWhetherAssignmentsHadNothingToReadWithoutAskingBrokers() + throws Exception { + var admin = new FakeAdmin(); + admin.highWatermark(PARTITION_0, 10L); + admin.highWatermark(PARTITION_1, 20L); + var checker = newReadinessChecker(admin); + KafkaReadPositions.assigned(PARTITION_0, 10L); + + assertThat(checker.assignmentsHaveNothingToRead(1000L)).isTrue(); + assertThat(checker.assignmentsHadNothingToReadAtLatestFetch()).isTrue(); + + KafkaReadPositions.assigned(PARTITION_1, 20L); + + assertThat(checker.assignmentsHadNothingToReadAtLatestFetch()).isFalse(); + assertThat(admin.listOffsetsCalls).hasSize(1); + } + @Test void testSplitRestoredBeforeTrailingCommitMarkerIsReadToEndOnceConsumerMovesPastIt() throws Exception { diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java index 4b619bb6..c734c6a1 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java @@ -21,6 +21,7 @@ import static com.datasqrl.flinkrunner.connector.kafka.SourceWatermarkOptions.OUT_OF_ORDERNESS_QUANTILE_DEFAULT; import static org.apache.flink.util.Preconditions.checkState; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.datasqrl.flinkrunner.connector.kafka.KafkaAdminIdleAdvanceReadinessChecker.LogEnd; import com.datasqrl.flinkrunner.connector.kafka.SourceWatermarkOptions.SourceWatermarkConfig; @@ -863,6 +864,7 @@ void testSubtaskReadingRowsIsNotReportedIdleBeforeItsFirstIdleWatermark() { // while no split has emitted a watermark yet, even when a split is reading rows. final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); final Reader reader = new Reader(source); + reader.initialAssignmentDone(); final ReaderSplit split = reader.newSplit(); reader.newSplit(); reader.tick(1); @@ -937,7 +939,8 @@ void testSubtaskReportedIdleBeforeItsSplitsWereAssignedTurnsActiveWhileOneAwaits @Test void testSubtaskReportedIdleBeforeItsSplitWasAssignedTurnsActiveBeforeBacklogProducesRow() { // Until the first row nothing else tells Flink that the subtask is active, with or without - // FLINK-40504, and downstream operators move their watermark past the backlog meanwhile. + // FLINK-40504. Turning active holds back a downstream operator's watermark only if no other + // subtask that feeds the operator has moved it yet; otherwise the backlog still arrives late. final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); final Reader reader = new Reader(source); reader.initialAssignmentDone(); @@ -960,6 +963,7 @@ void testSubtaskReadingBacklogIsNotReportedIdleWhileEmptySplitIsNotIdleYet() { // that it passes before the empty split's own, until which the splits have no watermark. final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); final Reader reader = new Reader(source); + reader.initialAssignmentDone(); reader.tick(3); final ReaderSplit backlog = reader.newSplit(); @@ -1000,9 +1004,10 @@ void testSubtaskTurnedActiveIsReportedIdleAgainOnceNothingIsLeftToRead() { } @Test - void testFailedBrokerCheckDoesNotTurnIdleSubtaskActive() { - // A check that fails, as when the brokers answer too slowly, says nothing about unread - // records. Turned active on it, a quiet source would hold back the watermark of a join. + void testIdleSubtaskDoesNotAskBrokers() { + // What turns it active needs no fetch: a newly assigned partition, or a partition's first row. + // So a broker check that fails, as when the brokers answer too slowly, cannot turn it active + // either, which would hold back the watermark of a join that a quiet source feeds. final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); final Reader reader = new Reader(source); reader.newSplit(); @@ -1012,11 +1017,42 @@ void testFailedBrokerCheckDoesNotTurnIdleSubtaskActive() { assertThat(reader.downstream).containsExactly("IDLE"); - source.broker.logAppendTime = false; - reader.tick(5); - source.broker.logAppendTime = true; + final int brokerChecks = source.broker.checks; reader.tick(10); + assertThat(source.broker.checks).isEqualTo(brokerChecks); + assertThat(reader.downstream).containsExactly("IDLE"); + } + + @Test + void testMainOutputGeneratorWithIdleAdvanceRejectsRows() { + // Kafka emits rows through split outputs. A row here means that Flink did not create the main + // output's generator first, and a split got the generator that emits no watermark. + final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); + final WatermarkGenerator mainOutputGenerator = + source.strategy().createWatermarkGenerator(context(SystemClock.getInstance())); + + assertThatThrownBy(() -> mainOutputGenerator.onEvent(null, 0L, new CollectingWatermarkOutput())) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void testRestoredSplitsReadToEndDoNotReportSubtaskIdleBeforeItsFirstAssignmentIsDone() { + // Flink checks the splits before the main output, which it counts as idle until it has + // reported. With a timeout below the watermark interval, splits marked idle on the first + // check would report the subtask idle before the main output has kept it active. + final Source source = + new Source(idleAdvanceConfig(MIN_RECORDS_DEFAULT, 1L, 100L), ONE_HOUR_MILLIS); + final Reader reader = new Reader(source); + reader.newSplit(); + reader.newSplit(); + reader.tick(3); + + assertThat(reader.downstream).isEmpty(); + + reader.initialAssignmentDone(); + reader.tick(2); + assertThat(reader.downstream).containsExactly("IDLE"); } @@ -1051,6 +1087,7 @@ void testSplitRestoredBeforeTrailingCommitMarkerDoesNotHoldBackSubtask() { // moves past the marker without returning a record: nothing is left to read. final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); final Reader reader = new Reader(source); + reader.initialAssignmentDone(); final ReaderSplit live = reader.newSplit(); final ReaderSplit restored = reader.newSplit(2L); restored.log().beginTransaction(); @@ -1069,12 +1106,17 @@ private static SourceWatermarkConfig idleAdvanceConfig(int minRecords) { private static SourceWatermarkConfig idleAdvanceConfig( int minRecords, long brokerCheckTtlMillis) { + return idleAdvanceConfig(minRecords, brokerCheckTtlMillis, IDLE_ADVANCE_TIMEOUT_MILLIS); + } + + private static SourceWatermarkConfig idleAdvanceConfig( + int minRecords, long brokerCheckTtlMillis, long idleAdvanceTimeoutMillis) { return new SourceWatermarkConfig( minRecords, MIN_OUT_OF_ORDERNESS_MILLIS_DEFAULT, MAX_OUT_OF_ORDERNESS_MILLIS_DEFAULT, OUT_OF_ORDERNESS_QUANTILE_DEFAULT, - IDLE_ADVANCE_TIMEOUT_MILLIS, + idleAdvanceTimeoutMillis, IDLE_ADVANCE_SAFETY_MARGIN_MILLIS, 1_000L, brokerCheckTtlMillis); @@ -1192,12 +1234,16 @@ private KafkaRecordTimestampWatermarkStrategy strategy() { return strategy; } - /** A split's generator, which Flink creates after the one of the reader's main output. */ + /** + * A split's generator, which Flink creates after the one of the reader's main output. The + * subtask's first assignment is done: no split is marked idle before. + */ private WatermarkGenerator newSplitGenerator(RelativeClock inputActivityClock) { if (!mainOutputGeneratorCreated) { strategy().createWatermarkGenerator(context(new PausableRelativeClock(monotonicClock))); mainOutputGeneratorCreated = true; } + KafkaReadPositions.initialAssignmentDone(); return strategy().createWatermarkGenerator(context(inputActivityClock)); } @@ -1541,12 +1587,16 @@ private static final class FakeBroker extends KafkaAdminIdleAdvanceReadinessChec private boolean logAppendTime = true; private boolean logEndsAvailable = true; + /** How often the brokers were asked. */ + private int checks; + private FakeBroker(SourceWatermarkConfig config, String sourceId) { super(KafkaFetchPositions.publishing(new Properties(), sourceId), List.of("topic"), config); } @Override boolean checkBrokerAndTopicTimestampType() { + checks++; return logAppendTime; } From de7f442b11946fc4754a2f4ac0501e77c6f2428d Mon Sep 17 00:00:00 2001 From: Mate Czagany Date: Tue, 17 Nov 2026 16:40:27 +0100 Subject: [PATCH 3/5] Address PR comments 2 --- README.md | 21 +++++++++----- .../kafka/AssignmentRecordingSource.java | 2 +- .../kafka/InitialAssignmentDoneEvent.java | 11 +------ .../InitialAssignmentSignalingEnumerator.java | 29 ++++++++++--------- ...KafkaAdminIdleAdvanceReadinessChecker.java | 21 +++++++------- .../connector/kafka/KafkaReadPositions.java | 5 ++-- ...KafkaRecordTimestampWatermarkStrategy.java | 16 +++++----- ...aAdminIdleAdvanceReadinessCheckerTest.java | 19 ++++++------ .../kafka/KafkaReadPositionsTest.java | 2 +- ...aRecordTimestampWatermarkStrategyTest.java | 4 +-- 10 files changed, 66 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 0e126063..de199b4e 100644 --- a/README.md +++ b/README.md @@ -341,17 +341,24 @@ A partition can be idle with records still unread, when the source is busy with For an idle partition, the connector asks the broker for the end of its log, and the watermark advances only if the partition has been read to that end. It then moves to the wall-clock time at which the broker was asked, minus the adaptive out-of-orderness delay and the safety margin, also when the partition's last record was read from a backlog, such as when a new job starts from the earliest offset. An idle partition with unread records, or whose log end cannot be fetched, keeps the watermark its records give it. -An idle partition that has not produced a row yet is marked idle instead, which leaves it out of the source watermark until it does, but only once every partition of the source subtask that has not produced a row has been read to the end of its log. -After a restore, a partition can take longer than the timeout to produce its first row, and marking it idle meanwhile would let the subtask's other partitions carry the watermark past its records. +An idle partition that has not produced a row yet is marked idle instead, which leaves it out of the source watermark until it does. +That happens only once every partition of the source subtask that has not produced a row has been read to the end of its log. +After a restore, a partition can take longer than the timeout to produce its first row. +Marking it idle meanwhile would let the subtask's other partitions carry the watermark past its records. Such a partition has been read to the end when its split started at the end of its log, or when its consumer has since reached that end, past any commit markers, and every record it returned has been read. On a fresh start from the earliest or the committed offsets, the connector first looks up where each split starts. -A partition's first row moves its watermark to just below that row at once, without waiting for `scan.source-watermark.min-records` records, so downstream operators see the source subtask as active, and hold their watermark below the row, before any watermark that would pass it. +A partition's first row moves its watermark to just below that row at once, without waiting for `scan.source-watermark.min-records` records. +Downstream operators then see the source subtask as active, and hold their watermark below the row, before any watermark that would pass it. A source subtask is reported idle only while none of its partitions has produced a row and each has been read to the end of its log, as far as the latest successful log-end fetch shows. -It is reported active again when a partition produces its first row, or before that when a new partition is assigned; while it is idle, it does not ask the brokers. -Turning active again holds back the watermark of a downstream operator only if no other source subtask that feeds the operator has moved it yet, since Flink leaves a subtask that turns active out of the operator's watermark until the subtask's own watermark has caught up with it. +It is reported active again when a partition produces its first row, or before that when a new partition is assigned. +While it is idle, it does not ask the brokers. +Turning active again holds back the watermark of a downstream operator only if no other source subtask that feeds the operator has moved it yet. +Flink leaves a subtask that turns active out of the operator's watermark until the subtask's own watermark has caught up with it. So with a source parallelism above one, the backlog of a partition that is discovered while the job runs can still arrive late. -A subtask is not reported idle before the enumerator has told it that its first partition discovery is done and assigned, since until then it may still be about to get a partition with a backlog, which downstream operators would move their watermark past meanwhile: its first on a fresh start, or after a restore one that was added to the topic while the job was down. -Until then it holds back the watermark of downstream operators, for as long as that first discovery takes. +A subtask is not reported idle before the source's enumerator, which assigns partitions to subtasks, has told it that its first partition discovery is done and assigned. +Until then the subtask may still be about to get a partition with a backlog: its first on a fresh start, or after a restore one that was added to the topic while the job was down. +Reported idle meanwhile, it would let downstream operators move their watermark past that backlog. +So until then it holds back the watermark of downstream operators, for as long as that first discovery takes. The log ends of all of a source subtask's partitions are fetched together, at most once per broker-check TTL, which therefore also bounds how far an idle partition's watermark trails the wall clock: keep it close to the idle-advance timeout. The safety margin must cover clock skew between the Kafka brokers and the TaskManagers, and the time a record takes to become readable after the broker timestamps it. diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSource.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSource.java index 718274c0..85b599e7 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSource.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/AssignmentRecordingSource.java @@ -82,7 +82,7 @@ public void addSplits(List splits) { @Override public void handleSourceEvents(SourceEvent sourceEvent) { if (sourceEvent instanceof InitialAssignmentDoneEvent) { - KafkaReadPositions.initialAssignmentDone(); + KafkaReadPositions.recordInitialAssignmentDone(); } else { super.handleSourceEvents(sourceEvent); } diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentDoneEvent.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentDoneEvent.java index 520d420b..9828630f 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentDoneEvent.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentDoneEvent.java @@ -15,7 +15,6 @@ */ package com.datasqrl.flinkrunner.connector.kafka; -import java.io.Serial; import org.apache.flink.annotation.Internal; import org.apache.flink.api.connector.source.SourceEvent; @@ -24,12 +23,4 @@ * the reader every split that discovery had for it, which may be none. */ @Internal -public final class InitialAssignmentDoneEvent implements SourceEvent { - - @Serial private static final long serialVersionUID = 1L; - - @Override - public String toString() { - return "InitialAssignmentDoneEvent"; - } -} +public record InitialAssignmentDoneEvent() implements SourceEvent {} diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java index 06083668..fc855a8a 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/InitialAssignmentSignalingEnumerator.java @@ -21,9 +21,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.connector.source.SourceSplit; import org.apache.flink.api.connector.source.SplitEnumerator; import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit; import org.apache.flink.util.function.FunctionWithException; /** @@ -41,11 +41,11 @@ *

If a handler of those calls fails, no reader gets the event; such a failure fails the job. */ @Internal -public final class InitialAssignmentSignalingEnumerator - extends ForwardingEnumerator { +public final class InitialAssignmentSignalingEnumerator + extends ForwardingEnumerator { - private final SplitEnumeratorContext context; - private final SplitEnumerator delegate; + private final SplitEnumeratorContext context; + private final SplitEnumerator delegate; /** The async calls of the first assignment whose handler has not returned yet. */ private int pendingInitialCalls; @@ -53,7 +53,6 @@ public final class InitialAssignmentSignalingEnumerator context, + SplitEnumeratorContext context, FunctionWithException< - SplitEnumeratorContext, SplitEnumerator, Exception> + SplitEnumeratorContext, + SplitEnumerator, + Exception> enumeratorFactory) throws Exception { this.context = checkNotNull(context, "Enumerator context must not be null."); @@ -74,7 +75,7 @@ public InitialAssignmentSignalingEnumerator( } @Override - protected SplitEnumerator delegate() { + protected SplitEnumerator delegate() { return delegate; } @@ -87,7 +88,7 @@ public void start() { inInitialAssignment = false; } - started = true; + // No handler has run yet: the coordinator runs them on this thread, after start() returns. signalIfInitialAssignmentDone(); } @@ -101,7 +102,7 @@ public void addReader(int subtaskId) { } private void signalIfInitialAssignmentDone() { - if (initialAssignmentDone || !started || pendingInitialCalls > 0) { + if (initialAssignmentDone || pendingInitialCalls > 0) { return; } @@ -115,7 +116,7 @@ private void signalIfInitialAssignmentDone() { * The context the wrapped enumerator works with: the coordinator's, except that it counts the * async calls of the first assignment. */ - private final class CallTrackingContext extends ForwardingEnumeratorContext { + private final class CallTrackingContext extends ForwardingEnumeratorContext { private CallTrackingContext() { super(context); @@ -143,9 +144,9 @@ private BiConsumer tracked(BiConsumer handler) { } pendingInitialCalls++; - var ran = new AtomicBoolean(); + var firstRunStarted = new AtomicBoolean(); return (result, error) -> { - if (ran.getAndSet(true)) { + if (firstRunStarted.getAndSet(true)) { handler.accept(result, error); return; } diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java index 9883ca88..068d5c3d 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessChecker.java @@ -145,25 +145,26 @@ && assignmentsWithoutRows(state).stream() * *

A failed broker check does not change the answer, as it says nothing about unread records. */ - public boolean assignmentsHaveNothingToRead(long currentTimeMillis) { - return assignmentsHaveNothingToRead(() -> refreshIfDue(currentTimeMillis)); + public boolean noAssignmentHasRowsOrUnreadRecords(long currentTimeMillis) { + return noAssignmentHasRowsOrUnreadRecords(() -> refreshIfDue(currentTimeMillis)); } /** - * {@link #assignmentsHaveNothingToRead(long)} as of the latest fetch, without asking the brokers. - * A subtask that is reported idle asks this, so that it leaves the brokers alone until it has - * something to read again. + * {@link #noAssignmentHasRowsOrUnreadRecords(long)} as of the latest fetch, without asking the + * brokers. A subtask that is reported idle asks this, so that it leaves the brokers alone until + * it has something to read again. */ - public boolean assignmentsHadNothingToReadAtLatestFetch() { - return assignmentsHaveNothingToRead(this::brokerState); + public boolean noAssignmentHadRowsOrUnreadRecordsAtLatestFetch() { + return noAssignmentHasRowsOrUnreadRecords(this::brokerState); } - private boolean assignmentsHaveNothingToRead(Supplier latestFetch) { + private boolean noAssignmentHasRowsOrUnreadRecords(Supplier latestFetch) { var assignments = KafkaReadPositions.assignments(); - var producedRow = brokerState().partitions; + var partitionsWithRows = brokerState().partitions; // A partition that has produced a row settles it without the brokers, which a subtask busy // reading would otherwise ask on its task thread every broker-check TTL. - if (assignments.stream().anyMatch(position -> producedRow.contains(position.partition()))) { + if (assignments.stream() + .anyMatch(position -> partitionsWithRows.contains(position.partition()))) { return false; } diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositions.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositions.java index d09d43be..63474e29 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositions.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositions.java @@ -39,7 +39,8 @@ * far the consumer has moved past them, {@link KafkaFetchPositions} learns. * *

A generator learns its partition only from its first row, so {@link AssignmentRecordingSource} - * also records here every split assigned to the reader, with the offset it starts at. + * also records here every split assigned to the reader, with the offset it starts at, and when the + * enumerator has finished the reader's first assignment. */ @Internal public final class KafkaReadPositions { @@ -91,7 +92,7 @@ public static void assigned(TopicPartition partition, long startingOffset) { * Records that the enumerator has assigned the reader on this thread every split its first * discovery had for it ({@link InitialAssignmentDoneEvent}). */ - public static void initialAssignmentDone() { + public static void recordInitialAssignmentDone() { FOR_THREAD.get().initialAssignmentDone = true; } diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java index 40823ba1..88407274 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java @@ -149,7 +149,7 @@ private enum Reported { private final RelativeClock inputActivityClock; /** Number of assigned partitions at the previous check that found nothing to read. */ - private int assignmentsAtLastIdleCheck; + private int assignmentCountAtLastIdleCheck; /** Input activity time since which the subtask has had nothing to read in its partitions. */ private long idleSinceInputActivityMillis; @@ -184,16 +184,16 @@ public void onPeriodicEmit(WatermarkOutput output) { long inputActivityMillis = inputActivityClock.relativeTimeMillis(); long currentTimeMillis = clock.currentTimeMillis(); - int assignments = KafkaReadPositions.assignments().size(); + int assignmentCount = KafkaReadPositions.assignments().size(); - if (assignments > 0 && !assignmentsHaveNothingToRead(currentTimeMillis)) { + if (assignmentCount > 0 && !noAssignmentHasRowsOrUnreadRecords(currentTimeMillis)) { idleSinceInputActivityMillis = inputActivityMillis; markActive(output); return; } - if (assignments != assignmentsAtLastIdleCheck) { - assignmentsAtLastIdleCheck = assignments; + if (assignmentCount != assignmentCountAtLastIdleCheck) { + assignmentCountAtLastIdleCheck = assignmentCount; idleSinceInputActivityMillis = inputActivityMillis; } @@ -207,12 +207,12 @@ public void onPeriodicEmit(WatermarkOutput output) { } } - private boolean assignmentsHaveNothingToRead(long currentTimeMillis) { + private boolean noAssignmentHasRowsOrUnreadRecords(long currentTimeMillis) { // A subtask reported idle does not ask the brokers again. What turns it active needs no new // fetch: a partition assigned since the latest one, or a partition's first row. return reported == Reported.IDLE - ? idleAdvanceReadinessChecker.assignmentsHadNothingToReadAtLatestFetch() - : idleAdvanceReadinessChecker.assignmentsHaveNothingToRead(currentTimeMillis); + ? idleAdvanceReadinessChecker.noAssignmentHadRowsOrUnreadRecordsAtLatestFetch() + : idleAdvanceReadinessChecker.noAssignmentHasRowsOrUnreadRecords(currentTimeMillis); } /** Marks the main output active, which keeps Flink from reporting the subtask idle. */ diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java index cfc05d42..ded06a38 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaAdminIdleAdvanceReadinessCheckerTest.java @@ -281,7 +281,7 @@ void testPartitionProducingRowsDoesNotHoldBackSplitsWithoutRows() throws Excepti void testSubtaskWithoutAssignmentsHasNothingToRead() throws Exception { var checker = newReadinessChecker(new FakeAdmin()); - assertThat(checker.assignmentsHaveNothingToRead(1000L)).isTrue(); + assertThat(checker.noAssignmentHasRowsOrUnreadRecords(1000L)).isTrue(); } @Test @@ -294,11 +294,11 @@ void testPartitionThatProducedRowKeepsSubtaskFromHavingNothingToRead() throws Ex KafkaReadPositions.assigned(PARTITION_0, 10L); KafkaReadPositions.assigned(PARTITION_1, 20L); - assertThat(checker.assignmentsHaveNothingToRead(1000L)).isTrue(); + assertThat(checker.noAssignmentHasRowsOrUnreadRecords(1000L)).isTrue(); checker.register(PARTITION_0); - assertThat(checker.assignmentsHaveNothingToRead(1000L)).isFalse(); + assertThat(checker.noAssignmentHasRowsOrUnreadRecords(1000L)).isFalse(); } @Test @@ -309,12 +309,13 @@ void testPartitionAssignedSinceLatestFetchKeepsSubtaskFromHavingNothingToRead() var checker = newReadinessChecker(admin); KafkaReadPositions.assigned(PARTITION_0, 10L); - assertThat(checker.assignmentsHaveNothingToRead(1000L)).isTrue(); + assertThat(checker.noAssignmentHasRowsOrUnreadRecords(1000L)).isTrue(); KafkaReadPositions.assigned(PARTITION_1, 20L); - assertThat(checker.assignmentsHaveNothingToRead(1001L)).isFalse(); - assertThat(checker.assignmentsHaveNothingToRead(1000L + BROKER_CHECK_TTL_MILLIS)).isTrue(); + assertThat(checker.noAssignmentHasRowsOrUnreadRecords(1001L)).isFalse(); + assertThat(checker.noAssignmentHasRowsOrUnreadRecords(1000L + BROKER_CHECK_TTL_MILLIS)) + .isTrue(); } @Test @@ -326,12 +327,12 @@ void testLatestFetchTellsWhetherAssignmentsHadNothingToReadWithoutAskingBrokers( var checker = newReadinessChecker(admin); KafkaReadPositions.assigned(PARTITION_0, 10L); - assertThat(checker.assignmentsHaveNothingToRead(1000L)).isTrue(); - assertThat(checker.assignmentsHadNothingToReadAtLatestFetch()).isTrue(); + assertThat(checker.noAssignmentHasRowsOrUnreadRecords(1000L)).isTrue(); + assertThat(checker.noAssignmentHadRowsOrUnreadRecordsAtLatestFetch()).isTrue(); KafkaReadPositions.assigned(PARTITION_1, 20L); - assertThat(checker.assignmentsHadNothingToReadAtLatestFetch()).isFalse(); + assertThat(checker.noAssignmentHadRowsOrUnreadRecordsAtLatestFetch()).isFalse(); assertThat(admin.listOffsetsCalls).hasSize(1); } diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositionsTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositionsTest.java index fdc8fc9b..220e32b9 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositionsTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaReadPositionsTest.java @@ -55,7 +55,7 @@ void testRecordsAssignedPartitionsWithWhereTheirSplitsStart() { void testRecordsThatInitialAssignmentIsDoneUntilCleared() { assertThat(KafkaReadPositions.isInitialAssignmentDone()).isFalse(); - KafkaReadPositions.initialAssignmentDone(); + KafkaReadPositions.recordInitialAssignmentDone(); assertThat(KafkaReadPositions.isInitialAssignmentDone()).isTrue(); diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java index c734c6a1..095b8c12 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java @@ -1243,7 +1243,7 @@ private WatermarkGenerator newSplitGenerator(RelativeClock inputActivit strategy().createWatermarkGenerator(context(new PausableRelativeClock(monotonicClock))); mainOutputGeneratorCreated = true; } - KafkaReadPositions.initialAssignmentDone(); + KafkaReadPositions.recordInitialAssignmentDone(); return strategy().createWatermarkGenerator(context(inputActivityClock)); } @@ -1487,7 +1487,7 @@ private Reader(Source source) { /** The enumerator's event that the reader's first assignment is done, as its reader records. */ private void initialAssignmentDone() { - KafkaReadPositions.initialAssignmentDone(); + KafkaReadPositions.recordInitialAssignmentDone(); } /** A split assigned at the start of its partition's log. */ From 6ed52edd78b674c0ce22847cc038f6a6eafe4315 Mon Sep 17 00:00:00 2001 From: Mate Czagany Date: Tue, 17 Nov 2026 16:50:57 +0100 Subject: [PATCH 4/5] Update README.md and Java docs --- README.md | 14 +++++++++----- .../KafkaRecordTimestampWatermarkStrategy.java | 10 +++++----- .../KafkaRecordTimestampWatermarkStrategyTest.java | 5 +++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index de199b4e..cca2c795 100644 --- a/README.md +++ b/README.md @@ -332,7 +332,8 @@ Set `scan.source-watermark.idle-advance-timeout` to advance the watermark conser ``` Idle advancement supports only the `topic` table option, not `topic-pattern`. Configuring `topic-pattern` with idle advancement will result in a failure. -The connector checks this through Kafka's `AdminClient` and requires permission to describe the topic configuration and its offsets, and on a fresh start from the committed offsets, also the consumer group. +It also needs the brokers to be reachable and the topic to use `LogAppendTime`. +The connector checks both through Kafka's `AdminClient` and requires permission to describe the topic configuration and its offsets, and on a fresh start from the committed offsets, also the consumer group. If Kafka cannot be reached or the topic is not configured for `LogAppendTime`, idle advancement pauses rather than advancing unsafely. Do not combine idle advancement with `scan.watermark.idle-timeout`. The connector ignores that option when idle advancement is enabled, because once Flink marks a partition idle, it stops letting the connector advance that partition's watermark. @@ -350,12 +351,15 @@ On a fresh start from the earliest or the committed offsets, the connector first A partition's first row moves its watermark to just below that row at once, without waiting for `scan.source-watermark.min-records` records. Downstream operators then see the source subtask as active, and hold their watermark below the row, before any watermark that would pass it. A source subtask is reported idle only while none of its partitions has produced a row and each has been read to the end of its log, as far as the latest successful log-end fetch shows. +It is reported idle once that has lasted for the idle-advance timeout. It is reported active again when a partition produces its first row, or before that when a new partition is assigned. While it is idle, it does not ask the brokers. -Turning active again holds back the watermark of a downstream operator only if no other source subtask that feeds the operator has moved it yet. -Flink leaves a subtask that turns active out of the operator's watermark until the subtask's own watermark has caught up with it. -So with a source parallelism above one, the backlog of a partition that is discovered while the job runs can still arrive late. -A subtask is not reported idle before the source's enumerator, which assigns partitions to subtasks, has told it that its first partition discovery is done and assigned. +While a subtask is idle, the watermark of a downstream operator follows the operator's other inputs, such as the source's other subtasks or the other table of a join, and it does not move back when the subtask turns active again. +So records that were written to a partition before the connector discovered it can still arrive late. +The connector looks for new partitions every `scan.topic-partition-discovery.interval`, 5 minutes by default. +Turning active again keeps the operator's watermark from moving further, but only if the source's other subtasks have not moved it yet. +Otherwise Flink leaves the subtask out of the operator's watermark until the subtask's own watermark has caught up with it. +A subtask is not reported idle, and none of its partitions is marked idle, before the source's enumerator, which assigns partitions to subtasks, has told it that its first partition discovery is done and assigned. Until then the subtask may still be about to get a partition with a backlog: its first on a fresh start, or after a restore one that was added to the topic while the job was down. Reported idle meanwhile, it would let downstream operators move their watermark past that backlog. So until then it holds back the watermark of downstream operators, for as long as that first discovery takes. diff --git a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java index 88407274..990fe475 100644 --- a/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java +++ b/connectors/kafka-safe-connector/src/main/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategy.java @@ -126,11 +126,11 @@ public WatermarkGenerator createWatermarkGenerator( * idle. * *

A subtask reported idle turns active again when it is assigned a partition the enumerator - * discovered later. That holds back the watermark of a downstream operator only if no other - * subtask that feeds the operator has moved it yet: Flink leaves a subtask that turns active out - * of the operator's watermark until the subtask's own watermark has caught up with it. So where - * other subtasks feed the same operator, as with a keyed join and a source parallelism above one, - * a backlog in that partition still arrives late. + * discovered later. That keeps the watermark of a downstream operator from moving further, but + * does not move it back: what the operator's other inputs carried it past while the subtask was + * idle still arrives late. And it holds only if the source's other subtasks have not moved that + * watermark yet. Otherwise Flink leaves a subtask that turns active out of the operator's + * watermark until the subtask's own watermark has caught up with it. */ private static final class SubtaskIdlenessGenerator implements WatermarkGenerator { diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java index 095b8c12..c76a6c54 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java @@ -939,8 +939,9 @@ void testSubtaskReportedIdleBeforeItsSplitsWereAssignedTurnsActiveWhileOneAwaits @Test void testSubtaskReportedIdleBeforeItsSplitWasAssignedTurnsActiveBeforeBacklogProducesRow() { // Until the first row nothing else tells Flink that the subtask is active, with or without - // FLINK-40504. Turning active holds back a downstream operator's watermark only if no other - // subtask that feeds the operator has moved it yet; otherwise the backlog still arrives late. + // FLINK-40504. Turning active keeps a downstream operator's watermark from moving further + // while the backlog is read, if the source's other subtasks have not moved it yet. What the + // watermark has passed already still arrives late. final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); final Reader reader = new Reader(source); reader.initialAssignmentDone(); From aeb5d243307cd9094495a3331619dd55e7818052 Mon Sep 17 00:00:00 2001 From: Ferenc Csaky Date: Tue, 17 Nov 2026 18:49:04 +0100 Subject: [PATCH 5/5] Test that a row produced without a watermark keeps the subtask active Signed-off-by: Ferenc Csaky --- ...aRecordTimestampWatermarkStrategyTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java index c76a6c54..0ef419f3 100644 --- a/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java +++ b/connectors/kafka-safe-connector/src/test/java/com/datasqrl/flinkrunner/connector/kafka/KafkaRecordTimestampWatermarkStrategyTest.java @@ -980,6 +980,28 @@ void testSubtaskReadingBacklogIsNotReportedIdleWhileEmptySplitIsNotIdleYet() { .containsExactly("record@0", "record@1", "record@2", "watermark@" + firstRowWatermark(0L)); } + @Test + void testPartitionThatProducedRowWithoutWatermarkKeepsSubtaskActive() { + // The first row arrives while the broker check fails, so its split emits no watermark for it, + // and Flink treats the splits as idle. Only the produced row then keeps the main output from + // reporting the subtask idle before the split's first idle watermark. + final Source source = new Source(IDLE_ADVANCE_CONFIG, ONE_HOUR_MILLIS); + final Reader reader = new Reader(source); + reader.initialAssignmentDone(); + final ReaderSplit rows = reader.newSplit(); + reader.newSplit(); + source.broker.logAppendTime = false; + reader.tick(1); + + rows.appendAndRead(source.wallClockMillis()); + source.broker.logAppendTime = true; + reader.tick(1); + rows.appendAndRead(source.wallClockMillis()); + reader.tick(10); + + assertThat(reader.downstream).doesNotContain("IDLE"); + } + @Test void testSubtaskTurnedActiveIsReportedIdleAgainOnceNothingIsLeftToRead() { // Every record of the split produces no row, so no watermark would end the subtask's wait.