From 9f019669e58f1b910b0f4f90c200f8b6ee5a3a35 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Fri, 16 Oct 2026 15:54:21 +0200 Subject: [PATCH 01/14] accept null metadata values in the opensearch-java bolts Metadata.setValue(key, null) stores a null element, and addValue keeps it next to the new value. StatusUpdaterBolt copied the values of every metadata key with List.of, and IndexerBolt the values of the keys with more than one value. List.of throws a NullPointerException on a null element; the bolts caught it and failed the tuple. Both bolts now copy the values with Arrays.asList, and the null is written as a JSON null inside the array, as the legacy module does. Signed-off-by: Davide Polato --- .../opensearch/bolt/IndexerBolt.java | 4 +- .../persistence/StatusUpdaterBolt.java | 4 +- .../bolt/NullMetadataValueTest.java | 146 ++++++++++++++++++ 3 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java index c98a0abab..f7a441855 100644 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java +++ b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java @@ -20,8 +20,8 @@ import static org.apache.stormcrawler.Constants.StatusStreamName; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.lang3.StringUtils; @@ -183,7 +183,7 @@ public void execute(Tuple tuple) { source.put(entry.getKey(), value); } } else if (entry.getValue().length > 1) { - source.put(entry.getKey(), List.of(entry.getValue())); + source.put(entry.getKey(), Arrays.asList(entry.getValue())); } } diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java index 86f373fee..2e0b92eb8 100644 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java +++ b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java @@ -18,9 +18,9 @@ package org.apache.stormcrawler.opensearch.persistence; import java.io.IOException; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -231,7 +231,7 @@ public void store( String[] values = metadata.getValues(mdKey); // periods are not allowed - replace with %2E mdKey = mdKey.replaceAll("\\.", "%2E"); - metadataMap.put(mdKey, List.of(values)); + metadataMap.put(mdKey, Arrays.asList(values)); } String partitionKey = partitioner.getPartition(url, metadata); diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java new file mode 100644 index 000000000..17f045226 --- /dev/null +++ b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.opensearch.bolt; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.topology.base.BaseRichBolt; +import org.apache.storm.tuple.Tuple; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.TestOutputCollector; +import org.apache.stormcrawler.TestUtil; +import org.apache.stormcrawler.indexing.AbstractIndexerBolt; +import org.apache.stormcrawler.opensearch.IndexCreation; +import org.apache.stormcrawler.opensearch.OpenSearchConnection; +import org.apache.stormcrawler.opensearch.persistence.StatusUpdaterBolt; +import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt; +import org.apache.stormcrawler.persistence.DefaultScheduler; +import org.apache.stormcrawler.persistence.Scheduler; +import org.apache.stormcrawler.persistence.Status; +import org.apache.stormcrawler.util.MetadataTransfer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; + +/** + * Metadata values can be null. Runs without OpenSearch: the static connection factory and index + * creation are mocked, and the bulk operation passed to the connection is inspected. + */ +class NullMetadataValueTest { + + private MockedStatic connectionFactory; + + private MockedStatic indexCreation; + + private OpenSearchConnection connection; + + private TestOutputCollector output; + + private BaseRichBolt bolt; + + @BeforeEach + void mockConnection() { + connection = mock(OpenSearchConnection.class); + connectionFactory = mockStatic(OpenSearchConnection.class); + connectionFactory + .when(() -> OpenSearchConnection.getConnection(any(), anyString(), any())) + .thenReturn(connection); + indexCreation = mockStatic(IndexCreation.class); + output = new TestOutputCollector(); + } + + @AfterEach + void cleanup() { + try { + // fails the tuples still waiting for a bulk response, so it runs after the assertions + bolt.cleanup(); + } finally { + indexCreation.close(); + connectionFactory.close(); + } + } + + private static Metadata metadataWithNullValue() { + Metadata md = new Metadata(); + md.setValue("key", null); + md.addValue("key", "value"); + return md; + } + + private static Tuple tuple() { + Tuple tuple = mock(Tuple.class); + when(tuple.getStringByField("url")).thenReturn("https://example.com/"); + when(tuple.getValueByField("status")).thenReturn(Status.FETCHED); + when(tuple.getValueByField("metadata")).thenReturn(metadataWithNullValue()); + return tuple; + } + + @SuppressWarnings("unchecked") + private Map sentDocument() { + ArgumentCaptor op = ArgumentCaptor.forClass(BulkOperation.class); + verify(connection).addToProcessor(op.capture()); + return (Map) op.getValue().index().document(); + } + + @Test + @SuppressWarnings("unchecked") + void statusUpdaterBoltSendsNullValue() { + Map conf = new HashMap<>(); + conf.put(Scheduler.schedulerClassParamName, DefaultScheduler.class.getName()); + conf.put( + AbstractStatusUpdaterBolt.cacheConfigParamName, + "maximumSize=10000,expireAfterAccess=1h"); + conf.put(MetadataTransfer.metadataPersistParamName, "key"); + bolt = new StatusUpdaterBolt(); + bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + + bolt.execute(tuple()); + + assertTrue(output.getFailedTuples().isEmpty()); + Map metadata = (Map) sentDocument().get("metadata"); + assertEquals(Arrays.asList(null, "value"), metadata.get("key")); + } + + @Test + void indexerBoltSendsNullValue() { + Map conf = new HashMap<>(); + conf.put(AbstractIndexerBolt.urlFieldParamName, "url"); + conf.put(AbstractIndexerBolt.metadata2fieldParamName, "key"); + bolt = new IndexerBolt("content"); + bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + + bolt.execute(tuple()); + + assertTrue(output.getFailedTuples().isEmpty()); + assertEquals(Arrays.asList(null, "value"), sentDocument().get("key")); + } +} From 30e8106114ca5d628ed67ad627e05674a1295f4a Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Fri, 16 Oct 2026 16:13:49 +0200 Subject: [PATCH 02/14] replace the legacy opensearch sources with the opensearch-java ones The files of external/opensearch/src that also exist in external/opensearch-java take the content of their opensearch-java counterparts in place, so git blame keeps the lines that did not change. The files that exist only in external/opensearch-java are moved. The following commits update the module's POM, configuration and README. Signed-off-by: Davide Polato --- .../BulkItemResponseToFailedFlag.java | 71 -- .../stormcrawler/opensearch/Constants.java | 23 - .../opensearch/DelegateRefresher.java | 181 ----- .../opensearch/IndexCreation.java | 116 --- .../opensearch/OpenSearchConnection.java | 516 ------------- .../stormcrawler/opensearch/WaitAckCache.java | 340 --------- .../opensearch/bolt/DeletionBolt.java | 186 ----- .../opensearch/bolt/IndexerBolt.java | 297 -------- .../filtering/JSONURLFilterWrapper.java | 83 -- .../opensearch/metrics/MetricsConsumer.java | 163 ---- .../opensearch/metrics/MetricsReporter.java | 219 ------ .../opensearch/metrics/StatusMetricsBolt.java | 185 ----- .../parse/filter/JSONResourceWrapper.java | 79 -- .../opensearch/persistence/AbstractSpout.java | 330 -------- .../persistence/AggregationSpout.java | 432 ----------- .../opensearch/persistence/HybridSpout.java | 263 ------- .../opensearch/persistence/QueueBolt.java | 191 ----- .../persistence/StatusUpdaterBolt.java | 364 --------- .../opensearch/DelegateRefresherTest.java | 256 ------- .../OpenSearchConnectionCredentialsTest.java | 128 ---- .../opensearch/WaitAckCacheTest.java | 336 -------- .../bolt/AbstractOpenSearchTest.java | 53 -- .../opensearch/bolt/IndexerBoltTest.java | 138 ---- .../opensearch/bolt/QueueBoltTest.java | 202 ----- .../opensearch/bolt/StatusBoltTest.java | 176 ----- .../metrics/MetricsReporterTest.java | 54 -- .../src/test/resources/indexer.mapping | 40 - .../src/test/resources/metrics.mapping | 40 - .../src/test/resources/queues.mapping | 33 - .../src/test/resources/status.mapping | 39 - .../opensearch/AsyncBulkProcessor.java | 0 .../BulkItemResponseToFailedFlag.java | 139 +--- .../opensearch/DelegateRefresher.java | 33 +- .../opensearch/IndexCreation.java | 84 +- .../opensearch/OpenSearchConnection.java | 715 +++++++++--------- .../stormcrawler/opensearch/WaitAckCache.java | 59 +- .../opensearch/bolt/DeletionBolt.java | 28 +- .../opensearch/bolt/IndexerBolt.java | 83 +- .../filtering/JSONURLFilterWrapper.java | 9 +- .../opensearch/metrics/MetricsConsumer.java | 33 +- .../opensearch/metrics/MetricsReporter.java | 32 +- .../opensearch/metrics/StatusMetricsBolt.java | 135 ++-- .../parse/filter/JSONResourceWrapper.java | 9 +- .../opensearch/persistence/AbstractSpout.java | 107 ++- .../persistence/AggregationSpout.java | 277 ++++--- .../opensearch/persistence/HybridSpout.java | 162 ++-- .../opensearch/persistence/QueueBolt.java | 41 +- .../persistence/StatusUpdaterBolt.java | 131 ++-- .../AsyncBulkProcessorHttpTest.java | 0 .../AsyncBulkProcessorRetryTest.java | 0 .../opensearch/AsyncBulkProcessorTest.java | 0 .../OpenSearchConnectionCredentialsTest.java | 199 ++--- .../opensearch/OpenSearchConnectionTest.java | 0 .../opensearch/WaitAckCacheTest.java | 155 ++-- .../bolt/AbstractOpenSearchTest.java | 3 +- .../bolt/NullMetadataValueTest.java | 0 .../opensearch/bolt/QueueBoltTest.java | 45 +- .../opensearch/bolt/StatusBoltTest.java | 41 +- .../persistence/AbstractSpoutTest.java | 0 59 files changed, 1335 insertions(+), 6719 deletions(-) delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java delete mode 100644 external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java delete mode 100644 external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java delete mode 100644 external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java delete mode 100644 external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java delete mode 100644 external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java delete mode 100644 external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java delete mode 100644 external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java delete mode 100644 external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java delete mode 100644 external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java delete mode 100644 external/opensearch-java/src/test/resources/indexer.mapping delete mode 100644 external/opensearch-java/src/test/resources/metrics.mapping delete mode 100644 external/opensearch-java/src/test/resources/queues.mapping delete mode 100644 external/opensearch-java/src/test/resources/status.mapping rename external/{opensearch-java => opensearch}/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java (100%) rename external/{opensearch-java => opensearch}/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java (100%) rename external/{opensearch-java => opensearch}/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java (100%) rename external/{opensearch-java => opensearch}/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java (100%) rename external/{opensearch-java => opensearch}/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java (100%) rename external/{opensearch-java => opensearch}/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java (100%) rename external/{opensearch-java => opensearch}/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java (100%) diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java deleted file mode 100644 index c32f162f2..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch; - -import java.util.Objects; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.opensearch.client.opensearch._types.ErrorCause; -import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; - -/** - * Wraps a {@link BulkResponseItem} with a pre-computed failure flag. A 409 (conflict) is not - * considered a failure — it simply indicates a document already existed when using create mode. - * - * @param response the original bulk response item - * @param failed whether this item represents a real failure (excludes 409 conflicts) - * @param id the document id from the response item - */ -public record BulkItemResponseToFailedFlag( - @NotNull BulkResponseItem response, boolean failed, @NotNull String id) { - - public BulkItemResponseToFailedFlag { - Objects.requireNonNull(response, "response"); - Objects.requireNonNull(id, "id"); - } - - /** Constructs with id derived from the response item. */ - public BulkItemResponseToFailedFlag(@NotNull BulkResponseItem response, boolean failed) { - this( - response, - failed, - Objects.requireNonNull(response.id(), "BulkResponseItem id must not be null")); - } - - /** Returns the error cause, or {@code null} if the item did not fail. */ - @Nullable - public ErrorCause getFailedCause() { - return response.error(); - } - - /** Returns a human-readable failure description, or {@code null} if the item did not fail. */ - @Nullable - public String getFailure() { - ErrorCause error = response.error(); - if (error == null) { - return null; - } - return error.reason() != null ? error.reason() : error.type(); - } - - // opensearch-java: status() returns int HTTP code, not RestStatus enum - /** Returns the HTTP status code of this response item. */ - public int getStatus() { - return response.status(); - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java deleted file mode 100644 index 8c0cbc989..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/Constants.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch; - -public interface Constants { - - String PARAMPREFIX = "opensearch."; -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java deleted file mode 100644 index fb03a9f0f..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch; - -import com.fasterxml.jackson.databind.JsonNode; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; -import org.apache.stormcrawler.JSONResource; -import org.opensearch.client.json.JsonData; -import org.opensearch.client.opensearch.OpenSearchClient; -import org.opensearch.client.opensearch.core.GetResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Loads a delegate class that implements both a required base type and {@link JSONResource}, then - * periodically refreshes its configuration from OpenSearch. Used by {@link - * org.apache.stormcrawler.opensearch.filtering.JSONURLFilterWrapper} and {@link - * org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper} to eliminate duplicated - * setup/refresh/cleanup logic. - * - *

This is the opensearch-java (OpenSearch Java Client 3.x / HC5) counterpart of the class with - * the same name in the {@code external/opensearch} module. It uses the typed {@link - * OpenSearchClient} instead of the deprecated {@code RestHighLevelClient}. - * - * @param the base type that the delegate must extend (e.g. URLFilter or ParseFilter) - */ -public class DelegateRefresher { - - private static final Logger LOG = LoggerFactory.getLogger(DelegateRefresher.class); - - private final T delegate; - private Timer refreshTimer; - private OpenSearchClient osClient; - - /** - * Creates a refresher by loading the delegate class from the JSON configuration. - * - * @param baseType the required base class (e.g. URLFilter.class or ParseFilter.class) - * @param stormConf the Storm configuration map - * @param filterParams the JSON params node containing "delegate" and optional "refresh" - * @param configurer callback to configure the delegate after instantiation - */ - public DelegateRefresher( - Class baseType, - Map stormConf, - JsonNode filterParams, - DelegateConfigure configurer) { - - JsonNode delegateNode = filterParams.get("delegate"); - if (delegateNode == null) { - throw new RuntimeException("delegateNode undefined!"); - } - - String delegateClassName = null; - JsonNode node = delegateNode.get("class"); - if (node != null && node.isTextual()) { - delegateClassName = node.asText(); - } - if (delegateClassName == null) { - throw new RuntimeException(baseType.getSimpleName() + " delegate class undefined!"); - } - - try { - Class filterClass = Class.forName(delegateClassName); - - if (!baseType.isAssignableFrom(filterClass)) { - throw new RuntimeException( - "Filter " + delegateClassName + " does not extend " + baseType.getName()); - } - - @SuppressWarnings("unchecked") - T instance = (T) filterClass.getDeclaredConstructor().newInstance(); - - if (!(instance instanceof JSONResource)) { - throw new RuntimeException( - "Filter " + delegateClassName + " does not implement JSONResource"); - } - - this.delegate = instance; - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - LOG.error("Can't setup {}: {}", delegateClassName, e); - throw new RuntimeException("Can't setup " + delegateClassName, e); - } - - // configure the delegate - JsonNode paramsNode = delegateNode.get("params"); - configurer.configure(delegate, stormConf, paramsNode); - - // set up periodic refresh from OpenSearch - int refreshRate = 600; - node = filterParams.get("refresh"); - if (node != null && (node.isInt() || node.isTextual())) { - refreshRate = node.asInt(refreshRate); - } - - final JSONResource resource = (JSONResource) delegate; - - refreshTimer = new Timer(); - refreshTimer.schedule( - new TimerTask() { - public void run() { - if (osClient == null) { - try { - osClient = OpenSearchConnection.getClient(stormConf, "config"); - } catch (Exception e) { - LOG.error("Exception while creating OpenSearch connection", e); - } - } - if (osClient != null) { - LOG.info("Reloading json resources from OpenSearch"); - try { - GetResponse response = - osClient.get( - g -> - g.index("config") - .id(resource.getResourceFile()), - JsonData.class); - if (response.found() && response.source() != null) { - String json = response.source().toJson().toString(); - resource.loadJSONResources( - new ByteArrayInputStream( - json.getBytes(StandardCharsets.UTF_8))); - } - } catch (Exception e) { - LOG.error("Can't load config from OpenSearch", e); - } - } - } - }, - refreshRate * 1000L, - refreshRate * 1000L); - } - - /** Returns the delegate instance. */ - public T getDelegate() { - return delegate; - } - - /** Cancels the refresh timer and closes the OpenSearch client. */ - public void cleanup() { - if (refreshTimer != null) { - refreshTimer.cancel(); - } - if (osClient != null) { - try { - osClient._transport().close(); - } catch (IOException e) { - LOG.error("Exception when closing OpenSearch client", e); - } - osClient = null; - } - } - - /** Callback interface for configuring the delegate after instantiation. */ - @FunctionalInterface - public interface DelegateConfigure { - void configure(T delegate, Map stormConf, JsonNode params); - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java deleted file mode 100644 index f76172057..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch; - -import com.google.common.io.Resources; -import java.io.IOException; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import org.opensearch.client.opensearch.OpenSearchClient; -import org.opensearch.client.opensearch.generic.Requests; -import org.opensearch.client.opensearch.generic.Response; -import org.opensearch.client.opensearch.indices.ExistsTemplateRequest; -import org.slf4j.Logger; - -public class IndexCreation { - - public static synchronized void checkOrCreateIndex( - OpenSearchClient client, String indexName, String boltType, Logger log) - throws IOException { - final boolean indexExists = client.indices().exists(req -> req.index(indexName)).value(); - log.info("Index '{}' exists? {}", indexName, indexExists); - // there's a possible check-then-update race condition - // createIndex intentionally catches and logs exceptions from OpenSearch - if (!indexExists) { - boolean created = - IndexCreation.createIndex(client, indexName, boltType + ".mapping", log); - log.info("Index '{}' created? {} using {}", indexName, created, boltType + ".mapping"); - } - } - - public static synchronized void checkOrCreateIndexTemplate( - OpenSearchClient client, String boltType, Logger log) throws IOException { - final String templateName = boltType + "-template"; - final boolean templateExists = - client.indices() - .existsTemplate(ExistsTemplateRequest.of(r -> r.name(templateName))) - .value(); - log.info("Template '{}' exists? {}", templateName, templateExists); - // there's a possible check-then-update race condition - // createTemplate intentionally catches and logs exceptions from OpenSearch - if (!templateExists) { - boolean created = - IndexCreation.createTemplate(client, templateName, boltType + ".mapping", log); - log.info("templateExists '{}' created? {}", templateName, created); - } - } - - private static boolean createTemplate( - OpenSearchClient client, String templateName, String resourceName, Logger log) { - - try { - final URL mapping = - Thread.currentThread().getContextClassLoader().getResource(resourceName); - - final String jsonIndexConfiguration = - Resources.toString(mapping, StandardCharsets.UTF_8); - - try (Response response = - client.generic() - .execute( - Requests.builder() - .endpoint("/_template/" + templateName) - .method("PUT") - .json(jsonIndexConfiguration) - .build())) { - int statusCode = response.getStatus(); - return statusCode == 200 || statusCode == 201; - } - } catch (Exception e) { - log.warn("template '{}' not created", templateName, e); - return false; - } - } - - private static boolean createIndex( - OpenSearchClient client, String indexName, String resourceName, Logger log) { - - try { - final URL mapping = - Thread.currentThread().getContextClassLoader().getResource(resourceName); - - final String jsonIndexConfiguration = - Resources.toString(mapping, StandardCharsets.UTF_8); - - try (Response response = - client.generic() - .execute( - Requests.builder() - .endpoint("/" + indexName) - .method("PUT") - .json(jsonIndexConfiguration) - .build())) { - int statusCode = response.getStatus(); - return statusCode == 200 || statusCode == 201; - } - } catch (Exception e) { - log.warn("index '{}' not created", indexName, e); - return false; - } - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java deleted file mode 100644 index eeb473628..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java +++ /dev/null @@ -1,516 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch; - -import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS; -import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS; - -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.regex.Pattern; -import javax.net.ssl.SSLContext; -import org.apache.commons.lang3.StringUtils; -import org.apache.hc.client5.http.auth.AuthScope; -import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; -import org.apache.hc.client5.http.config.ConnectionConfig; -import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; -import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; -import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; -import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; -import org.apache.hc.core5.http.HttpHost; -import org.apache.hc.core5.reactor.ssl.TlsDetails; -import org.apache.hc.core5.ssl.SSLContextBuilder; -import org.apache.hc.core5.util.Timeout; -import org.apache.stormcrawler.util.ConfUtils; -import org.jetbrains.annotations.NotNull; -import org.opensearch.client.json.jackson.JacksonJsonpMapper; -import org.opensearch.client.opensearch.OpenSearchAsyncClient; -import org.opensearch.client.opensearch.OpenSearchClient; -import org.opensearch.client.opensearch.core.bulk.BulkOperation; -import org.opensearch.client.transport.OpenSearchTransport; -import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utility class to instantiate an OpenSearch client and bulkprocessor based on the configuration. - */ -public final class OpenSearchConnection { - - private static final Logger LOG = LoggerFactory.getLogger(OpenSearchConnection.class); - - @NotNull private final OpenSearchClient client; - - @NotNull private final AsyncBulkProcessor processor; - - @NotNull private final OpenSearchTransport transport; - - private OpenSearchConnection( - @NotNull OpenSearchClient c, - @NotNull AsyncBulkProcessor p, - @NotNull OpenSearchTransport t) { - client = c; - processor = p; - transport = t; - } - - public OpenSearchClient getClient() { - return client; - } - - /** - * Creates a standalone {@link OpenSearchClient}. Used by classes that need a client without a - * bulk processor (e.g. spouts, filters). Callers are responsible for closing the returned - * client's transport via {@code client._transport().close()}. - */ - public static OpenSearchClient getClient(Map stormConf, String boltType) { - return buildClientResources(stormConf, boltType).client(); - } - - /** Adds a single bulk operation to the internal processor. */ - public void addToProcessor(final BulkOperation operation) { - processor.add(operation); - } - - /** - * Creates a connection with a default (no-op) listener. The values for bolt type are - * [indexer,status,metrics]. - */ - public static OpenSearchConnection getConnection( - Map stormConf, String boltType) { - AsyncBulkProcessor.Listener listener = - new AsyncBulkProcessor.Listener() { - @Override - public void afterBulk( - long arg0, - org.opensearch.client.opensearch.core.BulkRequest arg1, - org.opensearch.client.opensearch.core.BulkResponse arg2) {} - - @Override - public void afterBulk( - long arg0, - org.opensearch.client.opensearch.core.BulkRequest arg1, - Throwable arg2) {} - - @Override - public void beforeBulk( - long arg0, org.opensearch.client.opensearch.core.BulkRequest arg1) {} - }; - return getConnection(stormConf, boltType, listener); - } - - public static OpenSearchConnection getConnection( - Map stormConf, String boltType, AsyncBulkProcessor.Listener listener) { - - final String dottedType = boltType + "."; - - warnOnRemovedKeys(stormConf, dottedType); - - ClientResources cr = buildClientResources(stormConf, boltType); - - final String flushIntervalString = - ConfUtils.getString( - stormConf, Constants.PARAMPREFIX, dottedType, "flushInterval", "5s"); - - final long flushIntervalMillis = parseTimeValueToMillis(flushIntervalString, 5000); - - final int bulkActions = - ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "bulkActions", 50); - - final int concurrentRequests = - ConfUtils.getInt( - stormConf, Constants.PARAMPREFIX, dottedType, "concurrentRequests", 1); - - AsyncBulkProcessor bulkProcessor = null; - try { - bulkProcessor = - new AsyncBulkProcessor.Builder( - new OpenSearchAsyncClient(cr.transport()), listener) - .setBulkActions(bulkActions) - .setFlushIntervalMillis(flushIntervalMillis) - .setConcurrentRequests(concurrentRequests) - .build(); - - return new OpenSearchConnection(cr.client(), bulkProcessor, cr.transport()); - } catch (Exception e) { - if (bulkProcessor != null) { - try { - bulkProcessor.close(); - } catch (Exception suppressed) { - e.addSuppressed(suppressed); - } - } - try { - cr.transport().close(); - } catch (IOException suppressed) { - e.addSuppressed(suppressed); - } - throw e; - } - } - - private final AtomicBoolean isClosed = new AtomicBoolean(false); - - public void close() { - - if (!isClosed.compareAndSet(false, true)) { - LOG.warn("Tried to close an already closed connection!"); - return; - } - - LOG.debug("Start closing the OpenSearch connection"); - - // First, close the BulkProcessor ensuring pending actions are flushed - try { - boolean success = processor.awaitClose(60, TimeUnit.SECONDS); - if (!success) { - throw new RuntimeException( - "Failed to flush pending actions when closing BulkProcessor"); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - // Now close the transport (also shuts down the underlying HTTP client) - try { - transport.close(); - } catch (IOException e) { - LOG.trace("Transport threw IO exception on close."); - } - } - - /** - * Extracts the document ID from a {@link BulkOperation} regardless of its type (index, create, - * delete, update). - */ - public static String getBulkOperationId(BulkOperation op) { - if (op.isIndex()) { - return op.index().id(); - } - if (op.isCreate()) { - return op.create().id(); - } - if (op.isDelete()) { - return op.delete().id(); - } - if (op.isUpdate()) { - return op.update().id(); - } - return null; - } - - /** - * Returns the scopes the Basic credentials are registered for: the scheme, host and port of - * each configured address. A request to any other address does not receive them. - */ - static List credentialScopes(List hosts) { - final Set scopes = new LinkedHashSet<>(); - for (HttpHost host : hosts) { - scopes.add(new AuthScope(host)); - } - return new ArrayList<>(scopes); - } - - /** - * Returns the addresses which use plain http and are not a loopback address, i.e. those the - * Basic credentials would be sent to in the clear over the network. - */ - static List plainHttpHosts(List hosts) { - final List plain = new ArrayList<>(); - for (HttpHost host : hosts) { - if ("http".equalsIgnoreCase(host.getSchemeName()) && !isLoopback(host.getHostName())) { - plain.add(host); - } - } - return plain; - } - - private static final Pattern IPV4_LOOPBACK = Pattern.compile("127(\\.\\d{1,3}){3}"); - - /** Whether the host is localhost or a loopback IP literal. No name is resolved. */ - static boolean isLoopback(String hostname) { - String host = StringUtils.strip(hostname.toLowerCase(Locale.ROOT), "[]"); - return host.equals("localhost") - || IPV4_LOOPBACK.matcher(host).matches() - || host.equals("::1") - || host.equals("0:0:0:0:0:0:0:1"); - } - - private static void warnAboutPlainHttp(String boltType, List hosts) { - final List plain = plainHttpHosts(hosts); - if (!plain.isEmpty()) { - LOG.warn( - "OpenSearch credentials are configured for {} but the addresses {} use plain " - + "http, the credentials are sent unencrypted. Give the addresses an " - + "https:// scheme.", - boltType, - plain); - } - } - - // internal helpers - private record ClientResources(OpenSearchClient client, OpenSearchTransport transport) {} - - /** - * Logs a WARN for legacy configuration keys that are no longer honoured by this module, so that - * users migrating from {@code external/opensearch} notice silently-dropped tuning. See the - * module README for the full list of differences. - */ - private static void warnOnRemovedKeys(Map stormConf, String dottedType) { - final String responseBufferKey = Constants.PARAMPREFIX + dottedType + "responseBufferSize"; - if (stormConf.containsKey(responseBufferKey)) { - LOG.warn( - "Configuration key '{}' is set but no longer supported by the opensearch-java module. " - + "The HC5-based async transport does not expose an equivalent per-request " - + "heap-buffer override. The setting is ignored — remove it from your " - + "configuration. See external/opensearch-java/README.md for details.", - responseBufferKey); - } - - final String sniffKey = Constants.PARAMPREFIX + dottedType + "sniff"; - if (stormConf.containsKey(sniffKey)) { - LOG.warn( - "Configuration key '{}' is set but no longer supported by the opensearch-java module. " - + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so " - + "automatic node discovery is not available. Keep the 'addresses' list up to " - + "date manually or put a load balancer in front of the cluster. " - + "See external/opensearch-java/README.md for details.", - sniffKey); - } - } - - private static ClientResources buildClientResources( - Map stormConf, String boltType) { - - final String dottedType = boltType + "."; - - final List hosts = new ArrayList<>(); - - final List confighosts = - ConfUtils.loadListFromConf( - Constants.PARAMPREFIX, dottedType, "addresses", stormConf); - - // find ; separated values and tokenise as multiple addresses - // e.g. opensearch1:9200; opensearch2:9200 - if (confighosts.size() == 1) { - String input = confighosts.get(0); - confighosts.clear(); - confighosts.addAll(Arrays.asList(input.split(" *; *"))); - } - - for (String host : confighosts) { - // no port specified? use default one - int port = 9200; - String scheme = "http"; - // no scheme specified? use http - if (!host.startsWith(scheme)) { - host = "http://" + host; - } - URI uri = URI.create(host); - if (uri.getHost() == null) { - throw new RuntimeException("host undefined " + host); - } - if (uri.getPort() != -1) { - port = uri.getPort(); - } - if (uri.getScheme() != null) { - scheme = uri.getScheme(); - } - // HC5: constructor is (scheme, hostname, port) — not (hostname, port, scheme) - hosts.add(new HttpHost(scheme, uri.getHost(), port)); - } - - LOG.info( - "OpenSearch {} transport configured with {} host(s): {}", - boltType, - hosts.size(), - hosts); - - // authentication via user / password - final String user = - ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "user"); - final String password = - ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "password"); - - final String proxyhost = - ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "proxy.host"); - - final int proxyport = - ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "proxy.port", -1); - - final String proxyscheme = - ConfUtils.getString( - stormConf, Constants.PARAMPREFIX, dottedType, "proxy.scheme", "http"); - - final boolean disableTlsValidation = - ConfUtils.getBoolean( - stormConf, Constants.PARAMPREFIX, "", "disable.tls.validation", false); - - final boolean needsUser = StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password); - final boolean needsProxy = StringUtils.isNotBlank(proxyhost) && proxyport != -1; - - // Defaults from ApacheHttpClient5TransportBuilder (same as the former RestClientBuilder) - final int connectTimeout = - ConfUtils.getInt( - stormConf, - Constants.PARAMPREFIX, - dottedType, - "connect.timeout", - DEFAULT_CONNECT_TIMEOUT_MILLIS); - final int socketTimeout = - ConfUtils.getInt( - stormConf, - Constants.PARAMPREFIX, - dottedType, - "socket.timeout", - DEFAULT_RESPONSE_TIMEOUT_MILLIS); - - final boolean compression = - ConfUtils.getBoolean( - stormConf, Constants.PARAMPREFIX, dottedType, "compression", false); - - final ApacheHttpClient5TransportBuilder builder = - ApacheHttpClient5TransportBuilder.builder(hosts.toArray(new HttpHost[0])) - .setMapper(new JacksonJsonpMapper()); - - // Timeouts via ConnectionConfig on the builder's internal connection manager - builder.setConnectionConfigCallback( - connConfigBuilder -> - connConfigBuilder - .setConnectTimeout(Timeout.ofMilliseconds(connectTimeout)) - .setSocketTimeout(Timeout.ofMilliseconds(socketTimeout))); - - if (needsUser) { - warnAboutPlainHttp(boltType, hosts); - } - if (disableTlsValidation) { - LOG.warn( - "opensearch.disable.tls.validation is set: the certificates and host names of " - + "the OpenSearch nodes used for {} are not checked", - boltType); - } - - // Auth, proxy, and/or trust-all SSL via HttpClient customisation - if (needsUser || needsProxy || disableTlsValidation) { - builder.setHttpClientConfigCallback( - httpClientBuilder -> { - // hc.client5 auth: password is char[], AuthScope(host, port) - if (needsUser) { - final BasicCredentialsProvider credentialsProvider = - new BasicCredentialsProvider(); - final UsernamePasswordCredentials credentials = - new UsernamePasswordCredentials(user, password.toCharArray()); - for (AuthScope scope : credentialScopes(hosts)) { - credentialsProvider.setCredentials(scope, credentials); - } - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - } - // hc.client5 proxy: HttpHost(scheme, host, port) - if (needsProxy) { - httpClientBuilder.setProxy( - new HttpHost(proxyscheme, proxyhost, proxyport)); - } - // Custom connection manager overrides the builder's internal one, - // so timeouts and TlsDetailsFactory must be replicated here - if (disableTlsValidation) { - try { - final SSLContext sslContext = - SSLContextBuilder.create() - .loadTrustMaterial((chain, authType) -> true) - .build(); - httpClientBuilder.setConnectionManager( - PoolingAsyncClientConnectionManagerBuilder.create() - .setTlsStrategy( - ClientTlsStrategyBuilder.create() - .setSslContext(sslContext) - .setHostnameVerifier( - NoopHostnameVerifier - .INSTANCE) - // HTTP/2 ALPN negotiation - .setTlsDetailsFactory( - sslEngine -> - new TlsDetails( - sslEngine - .getSession(), - sslEngine - .getApplicationProtocol())) - .build()) - .setDefaultConnectionConfig( - ConnectionConfig.custom() - .setConnectTimeout( - Timeout.ofMilliseconds( - connectTimeout)) - .setSocketTimeout( - Timeout.ofMilliseconds( - socketTimeout)) - .build()) - .build()); - } catch (Exception e) { - throw new RuntimeException("Failed to disable TLS validation", e); - } - } - return httpClientBuilder; - }); - } - - // Compression: first-class builder method, not a request interceptor - builder.setCompressionEnabled(compression); - - final OpenSearchTransport transport = builder.build(); - final OpenSearchClient openSearchClient = new OpenSearchClient(transport); - - return new ClientResources(openSearchClient, transport); - } - - /** - * Parses a time value string (e.g. "5s", "500ms", "1m") into milliseconds. - * - * @param value the string to parse - * @param defaultMillis the default if parsing fails - * @return milliseconds - */ - static long parseTimeValueToMillis(String value, long defaultMillis) { - if (value == null || value.isBlank()) { - return defaultMillis; - } - value = value.trim(); - try { - if (value.endsWith("ms")) { - return Long.parseLong(value.substring(0, value.length() - 2)); - } else if (value.endsWith("s")) { - return Long.parseLong(value.substring(0, value.length() - 1)) * 1000; - } else if (value.endsWith("m")) { - return Long.parseLong(value.substring(0, value.length() - 1)) * 60000; - } else { - return Long.parseLong(value); - } - } catch (NumberFormatException e) { - LOG.warn("Could not parse time value '{}', using default {}ms", value, defaultMillis); - return defaultMillis; - } - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java deleted file mode 100644 index 0e8574fbb..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java +++ /dev/null @@ -1,340 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.RemovalCause; -import com.github.benmanes.caffeine.cache.Ticker; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import org.apache.storm.tuple.Tuple; -import org.apache.stormcrawler.metrics.ScopedCounter; -import org.jetbrains.annotations.Nullable; -// opensearch-java: uses typed BulkRequest/BulkResponse, not legacy REST equivalents -import org.opensearch.client.opensearch.core.BulkRequest; -import org.opensearch.client.opensearch.core.BulkResponse; -import org.slf4j.Logger; - -/** - * Thread-safe cache that tracks in-flight tuples awaiting bulk acknowledgment from OpenSearch. - * Provides shared logic for processing bulk responses and failing tuples on error, used by - * IndexerBolt, DeletionBolt, and StatusUpdaterBolt. - */ -public class WaitAckCache { - - /** Callback invoked for each tuple when processing a successful bulk response. */ - @FunctionalInterface - public interface TupleAction { - void handle(String id, Tuple tuple, BulkItemResponseToFailedFlag selected); - } - - private final Cache> cache; - private final ReentrantLock lock = new ReentrantLock(true); - private final Logger log; - private final Consumer onEviction; - - /** Creates a cache with a fixed 60-second expiry. */ - public WaitAckCache(Logger log, Consumer onEviction) { - this(Caffeine.newBuilder().expireAfterWrite(60, TimeUnit.SECONDS), log, onEviction); - } - - /** - * Creates a cache using a {@link Caffeine} spec string (e.g. {@code "expireAfterWrite=300s"}), - * typically driven by {@code topology.message.timeout.secs}. - */ - public WaitAckCache(String cacheSpec, Logger log, Consumer onEviction) { - this(Caffeine.from(cacheSpec), log, onEviction); - } - - /** Creates a cache with a custom ticker for deterministic time control in tests. */ - WaitAckCache(String cacheSpec, Logger log, Consumer onEviction, Ticker ticker) { - this(Caffeine.from(cacheSpec).ticker(ticker).executor(Runnable::run), log, onEviction); - } - - private WaitAckCache(Caffeine builder, Logger log, Consumer onEviction) { - this.log = log; - this.onEviction = onEviction; - this.cache = - builder.>removalListener( - (String key, List value, RemovalCause cause) -> { - if (!cause.wasEvicted()) { - return; - } - if (value != null) { - log.error( - "Purged from waitAck {} with {} values", - key, - value.size()); - for (Tuple t : value) { - onEviction.accept(t); - } - } else { - log.error("Purged from waitAck {} with no values", key); - } - }) - .build(); - } - - /** Returns the approximate number of entries in this cache. */ - public long estimatedSize() { - return cache.estimatedSize(); - } - - /** Adds a tuple to the cache under the given document ID, creating the list if needed. */ - public void addTuple(String docID, Tuple tuple) { - lock.lock(); - try { - List tt = cache.get(docID, k -> new LinkedList<>()); - tt.add(tuple); - if (log.isDebugEnabled()) { - String url = (String) tuple.getValueByField("url"); - log.debug("Added to waitAck {} with ID {} total {}", url, docID, tt.size()); - } - } finally { - lock.unlock(); - } - } - - /** Returns true if the cache contains an entry for the given document ID. */ - public boolean contains(String docID) { - lock.lock(); - try { - return cache.getIfPresent(docID) != null; - } finally { - lock.unlock(); - } - } - - /** Forces pending cache maintenance, triggering eviction listeners for expired entries. */ - public void cleanUp() { - cache.cleanUp(); - } - - /** Fails all remaining tuples in the cache and invalidates all entries. */ - public void shutdown() { - lock.lock(); - try { - Map> remaining = cache.asMap(); - for (var entry : remaining.entrySet()) { - log.warn( - "Shutdown: failing {} tuple(s) for ID {}", - entry.getValue().size(), - entry.getKey()); - for (Tuple t : entry.getValue()) { - onEviction.accept(t); - } - } - cache.invalidateAll(); - } finally { - lock.unlock(); - } - } - - /** Invalidates a single cache entry. */ - public void invalidate(String docID) { - lock.lock(); - try { - cache.invalidate(docID); - } finally { - lock.unlock(); - } - } - - /** - * Processes a successful bulk response: classifies each item (conflict vs failure), retrieves - * cached tuples, selects the best response per document ID, and invokes the action for each - * tuple. - * - * @param conflictCounter optional metric counter; if non-null, increments "doc_conflicts" scope - * for each conflict - */ - public void processBulkResponse( - BulkResponse response, - long executionId, - @Nullable ScopedCounter conflictCounter, - TupleAction action) { - - // opensearch-java: items() returns List; status() returns int - var idsToBulkItems = - response.items().stream() - .map( - bir -> { - var error = bir.error(); - boolean failed = false; - if (error != null) { - // opensearch-java: int status code, not RestStatus enum - if (bir.status() == 409) { - if (conflictCounter != null) { - conflictCounter.scope("doc_conflicts").incrBy(1); - } - log.debug("Doc conflict ID {}", bir.id()); - } else { - log.error( - "Bulk item failure ID {}: {}", - bir.id(), - error.reason() != null - ? error.reason() - : error.type()); - failed = true; - } - } - return new BulkItemResponseToFailedFlag(bir, failed); - }) - .collect( - // https://github.com/apache/stormcrawler/issues/832 - Collectors.groupingBy( - BulkItemResponseToFailedFlag::id, - Collectors.toUnmodifiableList())); - - Map> presentTuples; - long estimatedSize; - Set debugInfo = null; - lock.lock(); - try { - presentTuples = cache.getAllPresent(idsToBulkItems.keySet()); - if (!presentTuples.isEmpty()) { - cache.invalidateAll(presentTuples.keySet()); - } - estimatedSize = cache.estimatedSize(); - if (log.isDebugEnabled() && estimatedSize > 0L) { - debugInfo = new HashSet<>(cache.asMap().keySet()); - } - } finally { - lock.unlock(); - } - - int ackCount = 0; - int failureCount = 0; - - for (var entry : presentTuples.entrySet()) { - final var id = entry.getKey(); - final var tuples = entry.getValue(); - final var bulkItems = idsToBulkItems.get(id); - - BulkItemResponseToFailedFlag selected = selectBest(bulkItems, id); - - if (tuples != null) { - log.debug("Found {} tuple(s) for ID {}", tuples.size(), id); - for (Tuple t : tuples) { - if (selected.failed()) { - failureCount++; - } else { - ackCount++; - } - action.handle(id, t, selected); - } - } else { - log.warn("Could not find unacked tuples for {}", id); - } - } - - log.info( - "Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}", - executionId, - idsToBulkItems.size(), - estimatedSize, - ackCount, - failureCount); - - if (debugInfo != null) { - for (String k : debugInfo) { - log.debug("Still in wait ack after bulk response [{}] => {}", executionId, k); - } - } - } - - /** - * Processes a failed bulk request by failing all associated tuples. - * - * @param failAction callback applied to each tuple that must be failed - */ - public void processFailedBulk( - BulkRequest request, long executionId, Throwable failure, Consumer failAction) { - - log.error("Exception with bulk {} - failing the whole lot ", executionId, failure); - - // opensearch-java: operations() + getBulkOperationId replaces - // legacy requests() + DocWriteRequest::id - final var failedIds = - request.operations().stream() - .map(OpenSearchConnection::getBulkOperationId) - .filter(Objects::nonNull) - .collect(Collectors.toUnmodifiableSet()); - - Map> failedTupleLists; - lock.lock(); - try { - failedTupleLists = cache.getAllPresent(failedIds); - if (!failedTupleLists.isEmpty()) { - cache.invalidateAll(failedTupleLists.keySet()); - } - } finally { - lock.unlock(); - } - - for (var id : failedIds) { - var tuples = failedTupleLists.get(id); - if (tuples != null) { - log.debug("Failed {} tuple(s) for ID {}", tuples.size(), id); - for (Tuple t : tuples) { - failAction.accept(t); - } - } else { - log.warn("Could not find unacked tuple for {}", id); - } - } - } - - /** - * Selects the best response when there are multiple bulk items for the same document ID. - * Prefers non-failed responses; warns when there is a mix of success and failure. If all items - * are failed, returns the first one (no warning logged since there is no ambiguity). - */ - private BulkItemResponseToFailedFlag selectBest( - List items, String id) { - if (items.size() == 1) { - return items.get(0); - } - - BulkItemResponseToFailedFlag best = items.get(0); - int failedCount = 0; - for (var item : items) { - if (item.failed()) { - failedCount++; - } else { - best = item; - } - } - if (failedCount > 0 && failedCount < items.size()) { - log.warn( - "The id {} would result in an ack and a failure." - + " Using only the ack for processing.", - id); - } - return best; - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java deleted file mode 100644 index f7f0f1730..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.bolt; - -import java.lang.invoke.MethodHandles; -import java.util.Map; -import org.apache.commons.lang3.StringUtils; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.tuple.Tuple; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.indexing.AbstractIndexerBolt; -import org.apache.stormcrawler.metrics.CrawlerMetrics; -import org.apache.stormcrawler.opensearch.AsyncBulkProcessor; -import org.apache.stormcrawler.opensearch.OpenSearchConnection; -import org.apache.stormcrawler.opensearch.WaitAckCache; -import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.opensearch.core.BulkRequest; -import org.opensearch.client.opensearch.core.BulkResponse; -import org.opensearch.client.opensearch.core.bulk.BulkOperation; -import org.slf4j.LoggerFactory; - -/** - * Deletes documents in OpenSearch. This should be connected to the StatusUpdaterBolt via the - * 'deletion' stream and will remove the documents with a status of ERROR. Note that this component - * will also try to delete documents even though they were never indexed and it currently won't - * delete documents which were indexed under the canonical URL. - */ -public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener { - - static final org.slf4j.Logger LOG = - LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final String BOLT_TYPE = "indexer"; - - private OutputCollector _collector; - - private String indexName; - - private OpenSearchConnection connection; - - private WaitAckCache waitAck; - - private String docIdMetadataKey; - - public DeletionBolt() {} - - /** Sets the index name instead of taking it from the configuration. * */ - public DeletionBolt(String indexName) { - this.indexName = indexName; - } - - @Override - public void prepare( - Map conf, TopologyContext context, OutputCollector collector) { - _collector = collector; - if (indexName == null) { - indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content"); - } - - docIdMetadataKey = - ConfUtils.getString(conf, AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME); - - try { - connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this); - } catch (Exception e1) { - LOG.error("Can't connect to OpenSearch", e1); - throw new RuntimeException(e1); - } - - waitAck = new WaitAckCache(LOG, _collector::fail); - CrawlerMetrics.registerGauge(context, conf, "waitAck", waitAck::estimatedSize, 10); - } - - @Override - public void cleanup() { - waitAck.shutdown(); - if (connection != null) { - connection.close(); - } - } - - @Override - public void execute(Tuple tuple) { - String url = tuple.getStringByField("url"); - Metadata metadata = (Metadata) tuple.getValueByField("metadata"); - - // keep it simple for now and ignore cases where the canonical URL was - // used - - final String docID = getDocumentID(metadata, url); - final String targetIndex = getIndexName(metadata); - BulkOperation op = BulkOperation.of(b -> b.delete(d -> d.index(targetIndex).id(docID))); - - waitAck.addTuple(docID, tuple); - - try { - connection.addToProcessor(op); - } catch (Exception e) { - LOG.error("Error adding the deletion of {} to the bulk", url, e); - waitAck.invalidate(docID); - _collector.fail(tuple); - } - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer arg0) { - // none - } - - /** - * Must be overridden for implementing custom index names based on some metadata information By - * Default, indexName coming from config is used - */ - protected String getIndexName(Metadata m) { - return indexName; - } - - /** - * Get the document id. Uses the SHA-256 digest of the value found in the metadata for the key - * configured with {@link AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that - * the ID matches the one generated by the indexer for the same document. - * - * @param metadata The {@link Metadata}. - * @param url The normalised url. - * @return Return the SHA-256 digest of the value found in the metadata, or of the normalised - * url if not present. - */ - protected String getDocumentID(Metadata metadata, String url) { - String source = url; - if (docIdMetadataKey != null) { - final String fromMetadata = - StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey)); - if (fromMetadata != null) { - source = fromMetadata; - } - } - return org.apache.commons.codec.digest.DigestUtils.sha256Hex(source); - } - - @Override - public void beforeBulk(long executionId, BulkRequest request) {} - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - waitAck.processBulkResponse( - response, - executionId, - null, - (id, t, selected) -> { - if (!selected.failed()) { - _collector.ack(t); - } else { - String url = (String) t.getValueByField("url"); - LOG.error( - "update ID {}, URL {}, failure: {}", - id, - url, - selected.getFailure()); - _collector.fail(t); - } - }); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - waitAck.processFailedBulk(request, executionId, failure, _collector::fail); - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java deleted file mode 100644 index f7a441855..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.bolt; - -import static org.apache.stormcrawler.Constants.StatusStreamName; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import org.apache.commons.lang3.StringUtils; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; -import org.apache.stormcrawler.Constants; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.indexing.AbstractIndexerBolt; -import org.apache.stormcrawler.metrics.CrawlerMetrics; -import org.apache.stormcrawler.metrics.ScopedCounter; -import org.apache.stormcrawler.metrics.ScopedReducedMetric; -import org.apache.stormcrawler.opensearch.AsyncBulkProcessor; -import org.apache.stormcrawler.opensearch.IndexCreation; -import org.apache.stormcrawler.opensearch.OpenSearchConnection; -import org.apache.stormcrawler.opensearch.WaitAckCache; -import org.apache.stormcrawler.persistence.Status; -import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.opensearch.core.BulkRequest; -import org.opensearch.client.opensearch.core.BulkResponse; -import org.opensearch.client.opensearch.core.bulk.BulkOperation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Sends documents to opensearch. Indexes all the fields from the tuples or a Map - * <String,Object> from a named field. - */ -public class IndexerBolt extends AbstractIndexerBolt implements AsyncBulkProcessor.Listener { - - private static final Logger LOG = LoggerFactory.getLogger(IndexerBolt.class); - - private static final String OSBoltType = "indexer"; - - static final String OSIndexNameParamName = - org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".index.name"; - private static final String OSCreateParamName = - org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".create"; - private static final String OSIndexPipelineParamName = - org.apache.stormcrawler.opensearch.Constants.PARAMPREFIX + OSBoltType + ".pipeline"; - - private OutputCollector _collector; - - private String indexName; - - private String pipeline; - - // whether the document will be created only if it does not exist or - // overwritten - private boolean create = false; - - private ScopedCounter eventCounter; - - private OpenSearchConnection connection; - - private ScopedReducedMetric perSecMetrics; - - private WaitAckCache waitAck; - - public IndexerBolt() {} - - /** Sets the index name instead of taking it from the configuration. * */ - public IndexerBolt(String indexName) { - this.indexName = indexName; - } - - @Override - public void prepare( - Map conf, TopologyContext context, OutputCollector collector) { - super.prepare(conf, context, collector); - _collector = collector; - if (indexName == null) { - indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content"); - } - - create = ConfUtils.getBoolean(conf, IndexerBolt.OSCreateParamName, false); - pipeline = ConfUtils.getString(conf, IndexerBolt.OSIndexPipelineParamName); - - try { - connection = OpenSearchConnection.getConnection(conf, OSBoltType, this); - } catch (Exception e1) { - LOG.error("Can't connect to OpenSearch", e1); - throw new RuntimeException(e1); - } - - this.eventCounter = CrawlerMetrics.registerCounter(context, conf, "OpensearchIndexer", 10); - - this.perSecMetrics = - CrawlerMetrics.registerPerSecMetric(context, conf, "Indexer_average_persec", 10); - - waitAck = new WaitAckCache(LOG, _collector::fail); - CrawlerMetrics.registerGauge(context, conf, "waitAck", waitAck::estimatedSize, 10); - - // use the default status schema if none has been specified - try { - IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void cleanup() { - waitAck.shutdown(); - if (connection != null) { - connection.close(); - } - } - - @Override - public void execute(Tuple tuple) { - - final String url = tuple.getStringByField("url"); - - // Distinguish the value used for indexing - // from the one used for the status - final String normalisedurl = valueForURL(tuple); - - LOG.info("Indexing {} as {}", url, normalisedurl); - - final Metadata metadata = (Metadata) tuple.getValueByField("metadata"); - - if (!filterDocument(metadata)) { - LOG.info("Filtered {}", url); - eventCounter.scope("Filtered").incrBy(1); - // treat it as successfully processed even if - // we do not index it - _collector.emit(StatusStreamName, tuple, new Values(url, metadata, Status.FETCHED)); - _collector.ack(tuple); - return; - } - - final String docID = getDocumentID(metadata, normalisedurl); - - try { - final Map source = new HashMap<>(); - - // display text of the document? - if (StringUtils.isNotBlank(fieldNameForText())) { - final String text = trimText(tuple.getStringByField("text")); - if (!ignoreEmptyFields() || StringUtils.isNotBlank(text)) { - source.put(fieldNameForText(), trimText(text)); - } - } - - // send URL as field? - if (StringUtils.isNotBlank(fieldNameForURL())) { - source.put(fieldNameForURL(), normalisedurl); - } - - // which metadata to display? - final Map keyVals = filterMetadata(metadata); - - for (Entry entry : keyVals.entrySet()) { - if (entry.getValue().length == 1) { - final String value = entry.getValue()[0]; - if (!ignoreEmptyFields() || StringUtils.isNotBlank(value)) { - source.put(entry.getKey(), value); - } - } else if (entry.getValue().length > 1) { - source.put(entry.getKey(), Arrays.asList(entry.getValue())); - } - } - - final String targetIndex = getIndexName(metadata); - final BulkOperation op; - if (create) { - op = - BulkOperation.of( - b -> - b.create( - c -> { - c.index(targetIndex).id(docID).document(source); - if (pipeline != null) { - c.pipeline(pipeline); - } - return c; - })); - } else { - op = - BulkOperation.of( - b -> - b.index( - idx -> { - idx.index(targetIndex) - .id(docID) - .document(source); - if (pipeline != null) { - idx.pipeline(pipeline); - } - return idx; - })); - } - - waitAck.addTuple(docID, tuple); - - connection.addToProcessor(op); - - eventCounter.scope("Indexed").incrBy(1); - perSecMetrics.scope("Indexed").update(1); - } catch (Exception e) { - LOG.error("Error building document for OpenSearch", e); - // do not send to status stream so that it gets replayed - _collector.fail(tuple); - - waitAck.invalidate(docID); - } - } - - /** - * Must be overridden for implementing custom index names based on some metadata information By - * Default, indexName coming from config is used - */ - protected String getIndexName(Metadata m) { - return indexName; - } - - @Override - public void beforeBulk(long executionId, BulkRequest request) { - eventCounter.scope("bulks_sent").incrBy(1); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - eventCounter.scope("bulks_received").incrBy(1); - eventCounter.scope("bulk_msec").incrBy(response.took()); - - waitAck.processBulkResponse( - response, - executionId, - eventCounter, - (id, t, selected) -> { - String url = (String) t.getValueByField("url"); - Metadata metadata = (Metadata) t.getValueByField("metadata"); - - if (!selected.failed()) { - _collector.emit( - StatusStreamName, t, new Values(url, metadata, Status.FETCHED)); - _collector.ack(t); - } else { - var failure = selected.getFailure(); - LOG.error("update ID {}, URL {}, failure: {}", id, url, failure); - // there is something wrong with the content we should - // treat it as an ERROR - if (selected.getStatus() == 400) { - metadata.setValue(Constants.STATUS_ERROR_SOURCE, "OpenSearch indexing"); - metadata.setValue(Constants.STATUS_ERROR_MESSAGE, "invalid content"); - _collector.emit( - StatusStreamName, t, new Values(url, metadata, Status.ERROR)); - _collector.ack(t); - LOG.debug("Acked {} with ID {}", url, id); - } else { - _collector.fail(t); - LOG.debug("Failed {} with ID {}", url, id); - } - } - }); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - eventCounter.scope("bulks_received").incrBy(1); - waitAck.processFailedBulk( - request, - executionId, - failure, - t -> { - eventCounter.scope("failed").incrBy(1); - _collector.fail(t); - }); - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java deleted file mode 100644 index 49ee7f0ca..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.filtering; - -import com.fasterxml.jackson.databind.JsonNode; -import java.net.URL; -import java.util.Map; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.filtering.URLFilter; -import org.apache.stormcrawler.opensearch.DelegateRefresher; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -/** - * Wraps a URLFilter whose resources are in a JSON file that can be stored in OpenSearch. The - * benefit of doing this is that the resources can be refreshed automatically and modified without - * having to recompile the jar and restart the topology. The connection to OpenSearch is done via - * the config and uses a new bolt type 'config'. - * - *

The configuration of the delegate is done in the urlfilters.json as usual. - * - * 

- *  {
- *     "class": "org.apache.stormcrawler.opensearch.filtering.JSONURLFilterWrapper",
- *     "name": "OSFastURLFilter",
- *     "params": {
- *         "refresh": "60",
- *         "delegate": {
- *             "class": "org.apache.stormcrawler.filtering.regex.FastURLFilter",
- *             "params": {
- *                 "file": "fast.urlfilter.json"
- *             }
- *         }
- *     }
- *  }
- *
- * - * The resource file can be pushed to OpenSearch with - * - * 
- *  curl -XPUT 'localhost:9200/config/config/fast.urlfilter.json?pretty' -H 'Content-Type: application/json' -d @fast.urlfilter.json
- *
- */ -public class JSONURLFilterWrapper extends URLFilter { - - private DelegateRefresher refresher; - - public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) { - refresher = - new DelegateRefresher<>( - URLFilter.class, stormConf, filterParams, URLFilter::configure); - } - - @Override - public @Nullable String filter( - @Nullable URL sourceUrl, - @Nullable Metadata sourceMetadata, - @NotNull String urlToFilter) { - return refresher.getDelegate().filter(sourceUrl, sourceMetadata, urlToFilter); - } - - @Override - public void cleanup() { - if (refresher != null) { - refresher.cleanup(); - } - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java deleted file mode 100644 index c46a2b734..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.metrics; - -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import org.apache.storm.metric.api.IMetricsConsumer; -import org.apache.storm.task.IErrorReporter; -import org.apache.storm.task.TopologyContext; -import org.apache.stormcrawler.opensearch.IndexCreation; -import org.apache.stormcrawler.opensearch.OpenSearchConnection; -import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.opensearch.core.bulk.BulkOperation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Sends metrics to an OpenSearch index. The OpenSearch details are set in the configuration; an - * optional argument sets a date format to append to the index name. - * - * 
- *   topology.metrics.consumer.register:
- *        - class: "org.apache.stormcrawler.opensearch.metrics.MetricsConsumer"
- *          parallelism.hint: 1
- *          argument: "yyyy-MM-dd"
- *
- */ -public class MetricsConsumer implements IMetricsConsumer { - - private final Logger LOG = LoggerFactory.getLogger(getClass()); - - private static final String OSBoltType = "metrics"; - - /** name of the index to use for the metrics (default : metrics) * */ - private static final String OSMetricsIndexNameParamName = - "opensearch." + OSBoltType + ".index.name"; - - private String indexName; - - private OpenSearchConnection connection; - - private String stormID; - - /** optional date format passed as argument, must be parsable as a SimpleDateFormat */ - private SimpleDateFormat dateFormat; - - @Override - public void prepare( - Map stormConf, - Object registrationArgument, - TopologyContext context, - IErrorReporter errorReporter) { - indexName = ConfUtils.getString(stormConf, OSMetricsIndexNameParamName, "metrics"); - stormID = context.getStormId(); - if (registrationArgument != null) { - dateFormat = new SimpleDateFormat((String) registrationArgument, Locale.ROOT); - LOG.info("Using date format {}", registrationArgument); - } - try { - connection = OpenSearchConnection.getConnection(stormConf, OSBoltType); - } catch (Exception e1) { - LOG.error("Can't connect to OpenSearch", e1); - throw new RuntimeException(e1); - } - - // create a template if it doesn't exist - try { - IndexCreation.checkOrCreateIndexTemplate(connection.getClient(), OSBoltType, LOG); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void cleanup() { - if (connection != null) { - connection.close(); - } - } - - @Override - public void handleDataPoints(TaskInfo taskInfo, Collection dataPoints) { - final Date now = new Date(); - for (DataPoint dataPoint : dataPoints) { - handleDataPoints(taskInfo, dataPoint.name, dataPoint.value, now); - } - } - - private void handleDataPoints( - final TaskInfo taskInfo, final String nameprefix, final Object value, final Date now) { - if (value instanceof Number) { - indexDataPoint(taskInfo, now, nameprefix, ((Number) value).doubleValue()); - } else if (value instanceof Map) { - for (Entry entry : ((Map) value).entrySet()) { - String newnameprefix = nameprefix + "." + entry.getKey(); - handleDataPoints(taskInfo, newnameprefix, entry.getValue(), now); - } - } else if (value instanceof Collection) { - for (Object collectionObj : (Collection) value) { - handleDataPoints(taskInfo, nameprefix, collectionObj, now); - } - } else { - LOG.warn("Found data point value {} of {}", nameprefix, value.getClass().toString()); - } - } - - /** - * Returns the name of the index that metrics will be written to. - * - * @return elastic index name - */ - private String getIndexName(Date timestamp) { - if (dateFormat == null) { - return indexName; - } - - StringBuilder sb = new StringBuilder(indexName); - sb.append("-").append(dateFormat.format(timestamp)); - return sb.toString(); - } - - private void indexDataPoint(TaskInfo taskInfo, Date timestamp, String name, double value) { - try { - Map doc = new HashMap<>(); - doc.put("stormId", stormID); - doc.put("srcComponentId", taskInfo.srcComponentId); - doc.put("srcTaskId", taskInfo.srcTaskId); - doc.put("srcWorkerHost", taskInfo.srcWorkerHost); - doc.put("srcWorkerPort", taskInfo.srcWorkerPort); - doc.put("name", name); - doc.put("value", value); - doc.put("timestamp", timestamp.toInstant().toString()); - - final String targetIndex = getIndexName(timestamp); - BulkOperation op = - BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc))); - connection.addToProcessor(op); - } catch (Exception e) { - LOG.error("problem when building request for OpenSearch", e); - } - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java deleted file mode 100644 index d3b654a03..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.metrics; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Meter; -import com.codahale.metrics.MetricFilter; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; -import com.codahale.metrics.Timer; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.SortedMap; -import java.util.concurrent.TimeUnit; -import org.apache.storm.metrics2.reporters.ScheduledStormReporter; -import org.apache.stormcrawler.opensearch.IndexCreation; -import org.apache.stormcrawler.opensearch.OpenSearchConnection; -import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.opensearch.core.bulk.BulkOperation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Storm V2 metrics reporter that writes metrics to an OpenSearch index with the same document - * structure as the V1 {@link MetricsConsumer}. This allows existing OpenSearch dashboards to work - * unchanged during migration from V1 to V2 metrics. - * - *

Configuration in storm.yaml: - * - * 

- *   storm.metrics.reporters:
- *     - class: "org.apache.stormcrawler.opensearch.metrics.MetricsReporter"
- *       report.period: 10
- *       report.period.units: "SECONDS"
- *
- */ -public class MetricsReporter extends ScheduledStormReporter { - - private static final Logger LOG = LoggerFactory.getLogger(MetricsReporter.class); - - private static final String OSBoltType = "metrics"; - - private static final String OSMetricsIndexNameParamName = - "opensearch." + OSBoltType + ".index.name"; - - private static final String DATE_FORMAT_KEY = "opensearch.metrics.date.format"; - - private ScheduledReporter reporter; - - @Override - public void prepare( - MetricRegistry metricsRegistry, - Map topoConf, - Map reporterConf) { - - String indexName = ConfUtils.getString(topoConf, OSMetricsIndexNameParamName, "metrics"); - String stormId = (String) topoConf.getOrDefault("storm.id", "unknown"); - - SimpleDateFormat dateFormat = null; - String dateFormatStr = ConfUtils.getString(topoConf, DATE_FORMAT_KEY, null); - if (dateFormatStr != null) { - dateFormat = new SimpleDateFormat(dateFormatStr, Locale.ROOT); - } - - OpenSearchConnection connection; - try { - connection = OpenSearchConnection.getConnection(topoConf, OSBoltType); - } catch (Exception e) { - LOG.error("Can't connect to OpenSearch", e); - throw new RuntimeException(e); - } - - try { - IndexCreation.checkOrCreateIndexTemplate(connection.getClient(), OSBoltType, LOG); - } catch (IOException e) { - throw new RuntimeException(e); - } - - TimeUnit reportPeriodUnit = getReportPeriodUnit(reporterConf); - long reportPeriod = getReportPeriod(reporterConf); - - reporter = - new OpenSearchScheduledReporter( - metricsRegistry, indexName, stormId, dateFormat, connection); - - reporter.start(reportPeriod, reportPeriodUnit); - } - - @Override - public void start() { - // already started in prepare() - } - - @Override - public void stop() { - if (reporter != null) { - reporter.stop(); - } - } - - /** - * Inner ScheduledReporter that writes Codahale metrics to OpenSearch in the same format as the - * V1 {@link MetricsConsumer}. - */ - private static class OpenSearchScheduledReporter extends ScheduledReporter { - - private final String indexName; - private final String stormId; - private final SimpleDateFormat dateFormat; - private final OpenSearchConnection connection; - - OpenSearchScheduledReporter( - MetricRegistry registry, - String indexName, - String stormId, - SimpleDateFormat dateFormat, - OpenSearchConnection connection) { - super( - registry, - "opensearch-metrics-reporter", - MetricFilter.ALL, - TimeUnit.SECONDS, - TimeUnit.MILLISECONDS); - this.indexName = indexName; - this.stormId = stormId; - this.dateFormat = dateFormat; - this.connection = connection; - } - - @Override - @SuppressWarnings("rawtypes") - public void report( - SortedMap gauges, - SortedMap counters, - SortedMap histograms, - SortedMap meters, - SortedMap timers) { - - Date now = new Date(); - - for (Map.Entry entry : gauges.entrySet()) { - Object value = entry.getValue().getValue(); - if (value instanceof Number) { - indexDataPoint(now, entry.getKey(), ((Number) value).doubleValue()); - } else if (value instanceof Map) { - for (Map.Entry mapEntry : ((Map) value).entrySet()) { - if (mapEntry.getValue() instanceof Number) { - indexDataPoint( - now, - entry.getKey() + "." + mapEntry.getKey(), - ((Number) mapEntry.getValue()).doubleValue()); - } - } - } - } - - for (Map.Entry entry : counters.entrySet()) { - indexDataPoint(now, entry.getKey(), entry.getValue().getCount()); - } - - for (Map.Entry entry : histograms.entrySet()) { - indexDataPoint(now, entry.getKey(), entry.getValue().getSnapshot().getMean()); - } - - for (Map.Entry entry : meters.entrySet()) { - indexDataPoint(now, entry.getKey(), entry.getValue().getOneMinuteRate()); - } - - for (Map.Entry entry : timers.entrySet()) { - indexDataPoint(now, entry.getKey(), entry.getValue().getSnapshot().getMean()); - } - } - - private String getIndexName(Date timestamp) { - if (dateFormat == null) { - return indexName; - } - return indexName + "-" + dateFormat.format(timestamp); - } - - private void indexDataPoint(Date timestamp, String name, double value) { - try { - Map doc = new HashMap<>(); - doc.put("stormId", stormId); - doc.put("name", name); - doc.put("value", value); - doc.put("timestamp", timestamp.toInstant().toString()); - - final String targetIndex = getIndexName(timestamp); - BulkOperation op = - BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc))); - connection.addToProcessor(op); - } catch (Exception e) { - LOG.error("Problem when building request for OpenSearch", e); - } - } - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java deleted file mode 100644 index c74184c22..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.metrics; - -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.storm.Config; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.utils.TupleUtils; -import org.apache.stormcrawler.metrics.CrawlerMetrics; -import org.apache.stormcrawler.opensearch.Constants; -import org.apache.stormcrawler.opensearch.OpenSearchConnection; -import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.opensearch.OpenSearchClient; -import org.opensearch.client.opensearch._types.FieldValue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Queries the status index periodically to get the count of URLs per status. This bolt can be - * connected to the output of any other bolt and will not produce anything as output. - */ -public class StatusMetricsBolt extends BaseRichBolt { - - private static final Logger LOG = LoggerFactory.getLogger(StatusMetricsBolt.class); - - private static final String OSBoltType = "status"; - private static final String OSStatusIndexNameParamName = - Constants.PARAMPREFIX + "status.index.name"; - - private String indexName; - - private OpenSearchClient client; - - private Map latestStatusCounts = new ConcurrentHashMap<>(6); - - private int freqStats = 60; - - private OutputCollector _collector; - - private transient StatusCounter[] counters; - - private static final class StatusCounter { - final String name; - final AtomicBoolean ready = new AtomicBoolean(true); - - StatusCounter(String name) { - this.name = name; - } - } - - @Override - public void prepare( - Map stormConf, TopologyContext context, OutputCollector collector) { - _collector = collector; - indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status"); - try { - client = OpenSearchConnection.getClient(stormConf, OSBoltType); - } catch (Exception e1) { - LOG.error("Can't connect to OpenSearch", e1); - throw new RuntimeException(e1); - } - - CrawlerMetrics.registerGauge( - context, stormConf, "status.count", () -> latestStatusCounts, freqStats); - - counters = new StatusCounter[6]; - - counters[0] = new StatusCounter("DISCOVERED"); - counters[1] = new StatusCounter("FETCHED"); - counters[2] = new StatusCounter("FETCH_ERROR"); - counters[3] = new StatusCounter("REDIRECTION"); - counters[4] = new StatusCounter("ERROR"); - counters[5] = new StatusCounter("TOTAL"); - } - - @Override - public Map getComponentConfiguration() { - Config conf = new Config(); - conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, freqStats); - return conf; - } - - @Override - public void execute(Tuple input) { - _collector.ack(input); - - // this bolt can be connected to anything - // we just want to trigger a new search when the input is a tick tuple - if (!TupleUtils.isTick(input)) { - return; - } - - for (StatusCounter counter : counters) { - // still waiting for results from previous request - if (!counter.ready.compareAndSet(true, false)) { - LOG.debug("Not ready to get counts for status {}", counter.name); - continue; - } - final String statusName = counter.name; - CompletableFuture.supplyAsync( - () -> { - try { - if (statusName.equalsIgnoreCase("TOTAL")) { - return client.count(c -> c.index(indexName)); - } else { - return client.count( - c -> - c.index(indexName) - .query( - q -> - q.term( - t -> - t.field( - "status") - .value( - FieldValue - .of( - statusName))))); - } - } catch (Exception e) { - throw new CompletionException(e); - } - }) - .thenAccept( - response -> { - counter.ready.set(true); - LOG.debug( - "Got {} counts for status:{}", - response.count(), - statusName); - latestStatusCounts.put(statusName, response.count()); - }) - .exceptionally( - e -> { - counter.ready.set(true); - Throwable cause = - e instanceof CompletionException ? e.getCause() : e; - LOG.error( - "Failure when getting counts for status:{}", - statusName, - cause); - return null; - }); - } - } - - @Override - public void cleanup() { - if (client != null) { - try { - client._transport().close(); - } catch (Exception e) { - LOG.error("Exception closing client transport", e); - } - } - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - // NONE - THIS BOLT DOES NOT GET CONNECTED TO ANY OTHERS - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java deleted file mode 100644 index 6f6d7be66..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.parse.filter; - -import com.fasterxml.jackson.databind.JsonNode; -import java.util.Map; -import org.apache.stormcrawler.opensearch.DelegateRefresher; -import org.apache.stormcrawler.parse.ParseFilter; -import org.apache.stormcrawler.parse.ParseResult; -import org.jetbrains.annotations.NotNull; -import org.w3c.dom.DocumentFragment; - -/** - * Wraps a ParseFilter whose resources are in a JSON file that can be stored in OpenSearch. The - * benefit of doing this is that the resources can be refreshed automatically and modified without - * having to recompile the jar and restart the topology. The connection to OpenSearch is done via - * the config and uses a new bolt type 'config'. - * - *

The configuration of the delegate is done in the parsefilters.json as usual. - * - * 

- *  {
- *     "class": "org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper",
- *     "name": "OpenSearchCollectionTagger",
- *     "params": {
- *         "refresh": "60",
- *         "delegate": {
- *             "class": "org.apache.stormcrawler.parse.filter.CollectionTagger",
- *             "params": {
- *                 "file": "collections.json"
- *             }
- *         }
- *     }
- *  }
- *
- * - * The resource file can be pushed to OpenSearch with - * - * 
- *  curl -XPUT "$OSHOST/config/_create/collections.json" -H 'Content-Type: application/json' -d @src/main/resources/collections.json
- *
- */ -public class JSONResourceWrapper extends ParseFilter { - - private DelegateRefresher refresher; - - public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) { - refresher = - new DelegateRefresher<>( - ParseFilter.class, stormConf, filterParams, ParseFilter::configure); - } - - @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { - refresher.getDelegate().filter(url, content, doc, parse); - } - - @Override - public void cleanup() { - if (refresher != null) { - refresher.cleanup(); - } - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java deleted file mode 100644 index 0afe2b194..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.persistence; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.opensearch.Constants; -import org.apache.stormcrawler.opensearch.IndexCreation; -import org.apache.stormcrawler.opensearch.OpenSearchConnection; -import org.apache.stormcrawler.persistence.AbstractQueryingSpout; -import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.json.JsonData; -import org.opensearch.client.opensearch.OpenSearchClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AbstractSpout extends AbstractQueryingSpout { - - private static final Logger LOG = LoggerFactory.getLogger(AbstractSpout.class); - - protected static final String OSBoltType = "status"; - protected static final String OSStatusIndexNameParamName = - Constants.PARAMPREFIX + OSBoltType + ".index.name"; - - /** Field name to use for aggregating * */ - protected static final String OSStatusBucketFieldParamName = - Constants.PARAMPREFIX + OSBoltType + ".bucket.field"; - - protected static final String OSStatusMaxBucketParamName = - Constants.PARAMPREFIX + OSBoltType + ".max.buckets"; - protected static final String OSStatusMaxURLsParamName = - Constants.PARAMPREFIX + OSBoltType + ".max.urls.per.bucket"; - - /** Field name to use for sorting the URLs within a bucket, not used if empty or null. */ - protected static final String OSStatusBucketSortFieldParamName = - Constants.PARAMPREFIX + OSBoltType + ".bucket.sort.field"; - - /** Field name to use for sorting the buckets, not used if empty or null. */ - protected static final String OSStatusGlobalSortFieldParamName = - Constants.PARAMPREFIX + OSBoltType + ".global.sort.field"; - - protected static final String OSStatusFilterParamName = - Constants.PARAMPREFIX + OSBoltType + ".filterQuery"; - - protected static final String OSStatusQueryTimeoutParamName = - Constants.PARAMPREFIX + OSBoltType + ".query.timeout"; - - /** Query to use as a positive filter, set by es.status.filterQuery */ - protected List filterQueries = null; - - protected String indexName; - - protected static OpenSearchClient client; - - /** - * when using multiple instances - each one is in charge of a specific shard useful when - * sharding based on host or domain to guarantee a good mix of URLs - */ - protected int shardID = -1; - - /** Used to distinguish between instances in the logs * */ - protected String logIdprefix = ""; - - /** Field name used for field collapsing e.g. key * */ - protected String partitionField; - - protected int maxURLsPerBucket = 10; - - protected int maxBucketNum = 10; - - protected List bucketSortField = new ArrayList<>(); - - protected String totalSortField = ""; - - protected Date queryDate; - - protected int queryTimeout = -1; - - /** - * Single-threaded executor dedicated to running the asynchronous OpenSearch queries triggered - * from this spout. Using a dedicated, named, daemon executor avoids polluting the JVM-wide - * {@link java.util.concurrent.ForkJoinPool#commonPool() common pool} (which is shared with all - * other Storm tasks running in the same worker) and allows clean shutdown when the topology is - * stopped. - */ - protected ExecutorService queryExecutor; - - @Override - public void open( - Map stormConf, - TopologyContext context, - SpoutOutputCollector collector) { - - super.open(stormConf, context, collector); - - final String executorName = - "OpenSearchSpout-" - + context.getThisComponentId() - + "-" - + context.getThisTaskIndex(); - queryExecutor = - Executors.newSingleThreadExecutor( - r -> { - Thread t = new Thread(r, executorName); - t.setDaemon(true); - return t; - }); - - indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status"); - - // one OS client per JVM - synchronized (AbstractSpout.class) { - try { - if (client == null) { - client = OpenSearchConnection.getClient(stormConf, OSBoltType); - } - } catch (Exception e1) { - LOG.error("Can't connect to OpenSearch", e1); - throw new RuntimeException(e1); - } - - // use the default status schema if none has been specified - try { - IndexCreation.checkOrCreateIndex(client, indexName, OSBoltType, LOG); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - // if more than one instance is used we expect their number to be the - // same as the number of shards - int totalTasks = context.getComponentTasks(context.getThisComponentId()).size(); - if (totalTasks > 1) { - logIdprefix = - "[" + context.getThisComponentId() + " #" + context.getThisTaskIndex() + "] "; - - // determine the number of shards so that we can restrict the - // search - - // TODO use the admin API when it gets available - // TODO or the low level one with - // https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-shards-stores.html - // TODO identify local shards and use those if possible - - // ClusterSearchShardsRequest request = new - // ClusterSearchShardsRequest( - // indexName); - // ClusterSearchShardsResponse shardresponse = client.admin() - // .cluster().searchShards(request).actionGet(); - // ClusterSearchShardsGroup[] shardgroups = - // shardresponse.getGroups(); - // if (totalTasks != shardgroups.length) { - // throw new RuntimeException( - // "Number of OS spout instances should be the same as number of - // shards (" - // + shardgroups.length + ") but is " + totalTasks); - // } - // shardID = shardgroups[context.getThisTaskIndex()].getShardId() - // .getId(); - - // TEMPORARY simply use the task index as shard index - shardID = context.getThisTaskIndex(); - LOG.info("{} assigned shard ID {}", logIdprefix, shardID); - } - - partitionField = ConfUtils.getString(stormConf, OSStatusBucketFieldParamName, "key"); - - bucketSortField = ConfUtils.loadListFromConf(OSStatusBucketSortFieldParamName, stormConf); - - totalSortField = ConfUtils.getString(stormConf, OSStatusGlobalSortFieldParamName); - - maxURLsPerBucket = ConfUtils.getInt(stormConf, OSStatusMaxURLsParamName, 1); - maxBucketNum = ConfUtils.getInt(stormConf, OSStatusMaxBucketParamName, 10); - - queryTimeout = ConfUtils.getInt(stormConf, OSStatusQueryTimeoutParamName, -1); - - filterQueries = ConfUtils.loadListFromConf(OSStatusFilterParamName, stormConf); - } - - /** Builds a query and use it retrieve the results from OS * */ - protected abstract void populateBuffer(); - - /** - * Converts the {@link JsonData} carried by a search hit into a plain {@code Map} populated with native Java types (strings, numbers, lists, nested maps). - * - *

This helper exists to work around a subtle short-circuit inside {@code JsonDataImpl#to}: - * - * 

{@code
-     * if (clazz.isAssignableFrom(value.getClass())) {
-     *     return (T) value;
-     * }
-     * }
- * - *

When {@link JsonData} is produced via {@code JsonData._DESERIALIZER} (as opposed to a raw - * Jackson {@code TreeNode}), its internal {@code value} is a Jakarta JSON-P {@code JsonObject} - * — typically a Parsson {@code JsonObjectImpl}. Crucially, {@code jakarta.json.JsonObject - * extends Map}, so calling {@code source.to(Map.class)} or {@code - * source.to(Object.class)} matches the {@code isAssignableFrom} guard and returns the raw - * {@code JsonObject} without ever consulting the configured {@link - * org.opensearch.client.json.jackson.JacksonJsonpMapper}. The resulting map values are - * still {@code JsonString}/{@code JsonStringImpl} instances and any downstream {@code (String) - * value} cast blows up with a {@link ClassCastException}. - * - *

Passing a concrete subclass that {@code JsonObject} does not implement (here - * {@link LinkedHashMap}) bypasses the short-circuit and forces the round-trip through {@code - * JacksonJsonpGenerator -> JacksonJsonpParser -> Jackson ObjectMapper.readValue}, which yields - * native Java values (String, Long, Double, ArrayList, LinkedHashMap…). - * - * @param source the document source carried by an OpenSearch hit; may be {@code null} - * @return a (possibly empty) map of native Java values, never {@code null} - */ - static Map sourceAsMap(JsonData source) { - if (source == null) { - return Collections.emptyMap(); - } - @SuppressWarnings("unchecked") - Map map = source.to(LinkedHashMap.class); - return map == null ? Collections.emptyMap() : map; - } - - /** - * Adds a document source to the URL buffer unless it is already being processed. - * - * @param source the document source as a key-value map (must contain a "url" entry) - * @return {@code true} if the URL was added to the buffer, {@code false} if it was already - * being processed or already present - */ - protected final boolean addHitToBuffer(Map source) { - String url = (String) source.get("url"); - // is already being processed - skip it! - if (beingProcessed.containsKey(url)) { - return false; - } - return buffer.add(url, fromKeyValues(source)); - } - - protected final Metadata fromKeyValues(Map keyValues) { - Map> mdAsMap = (Map>) keyValues.get("metadata"); - Metadata metadata = new Metadata(); - if (mdAsMap != null) { - for (Entry> mdEntry : mdAsMap.entrySet()) { - String key = mdEntry.getKey(); - // periods are not allowed - replace with %2E - key = key.replaceAll("%2E", "\\."); - Object mdValObj = mdEntry.getValue(); - // single value - if (mdValObj instanceof String) { - metadata.addValue(key, (String) mdValObj); - } else { - // multi valued - metadata.addValues(key, (List) mdValObj); - } - } - } - return metadata; - } - - @Override - public void ack(Object msgId) { - LOG.debug("{} Ack for {}", logIdprefix, msgId); - super.ack(msgId); - } - - @Override - public void fail(Object msgId) { - LOG.info("{} Fail for {}", logIdprefix, msgId); - super.fail(msgId); - } - - @Override - public void close() { - // Shut the dedicated query executor down first so no more requests can be enqueued - // against the client while we are tearing it down. We give in-flight queries a short - // grace period to drain to keep shutdown reasonably graceful. - if (queryExecutor != null) { - queryExecutor.shutdown(); - try { - if (!queryExecutor.awaitTermination(30, TimeUnit.SECONDS)) { - LOG.warn( - "{} Query executor did not terminate within 30s, forcing shutdown", - logIdprefix); - queryExecutor.shutdownNow(); - } - } catch (InterruptedException e) { - queryExecutor.shutdownNow(); - Thread.currentThread().interrupt(); - } - queryExecutor = null; - } - - synchronized (AbstractSpout.class) { - if (client != null) { - try { - client._transport().close(); - } catch (IOException e) { - LOG.error("Exception caught when closing client", e); - } - client = null; - } - } - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java deleted file mode 100644 index a6272f368..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java +++ /dev/null @@ -1,432 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.persistence; - -import java.io.IOException; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import org.apache.commons.lang3.StringUtils; -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.opensearch.Constants; -import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.json.JsonData; -import org.opensearch.client.opensearch._types.FieldValue; -import org.opensearch.client.opensearch._types.SortOrder; -import org.opensearch.client.opensearch._types.aggregations.Aggregate; -import org.opensearch.client.opensearch._types.aggregations.Aggregation; -import org.opensearch.client.opensearch._types.aggregations.StringTermsBucket; -import org.opensearch.client.opensearch._types.aggregations.TopHitsAggregate; -import org.opensearch.client.opensearch.core.SearchRequest; -import org.opensearch.client.opensearch.core.SearchResponse; -import org.opensearch.client.opensearch.core.search.Hit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Spout which pulls URL from an OpenSearch index. Use a single instance unless you use - * 'opensearch.status.routing' with the StatusUpdaterBolt, in which case you need to have exactly - * the same number of spout instances as OpenSearch shards. Guarantees a good mix of URLs by - * aggregating them by an arbitrary field e.g. key. - */ -public class AggregationSpout extends AbstractSpout { - - private static final Logger LOG = LoggerFactory.getLogger(AggregationSpout.class); - - private static final String StatusSampleParamName = Constants.PARAMPREFIX + "status.sample"; - private static final String MostRecentDateIncreaseParamName = - Constants.PARAMPREFIX + "status.recentDate.increase"; - private static final String MostRecentDateMinGapParamName = - Constants.PARAMPREFIX + "status.recentDate.min.gap"; - - private boolean sample = false; - - private int recentDateIncrease = -1; - private int recentDateMinGap = -1; - - protected Set currentBuckets; - - @Override - public void open( - Map stormConf, - TopologyContext context, - SpoutOutputCollector collector) { - sample = ConfUtils.getBoolean(stormConf, StatusSampleParamName, sample); - recentDateIncrease = - ConfUtils.getInt(stormConf, MostRecentDateIncreaseParamName, recentDateIncrease); - recentDateMinGap = - ConfUtils.getInt(stormConf, MostRecentDateMinGapParamName, recentDateMinGap); - super.open(stormConf, context, collector); - currentBuckets = new HashSet<>(); - } - - @Override - protected void populateBuffer() { - - if (queryDate == null) { - queryDate = new Date(); - lastTimeResetToNow = Instant.now(); - } - - String formattedQueryDate = - Instant.ofEpochMilli(queryDate.getTime()) - .atOffset(ZoneOffset.UTC) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); - - LOG.info("{} Populating buffer with nextFetchDate <= {}", logIdprefix, formattedQueryDate); - - // Build the top_hits sub-aggregation - Aggregation topHitsAgg = - Aggregation.of( - a -> - a.topHits( - th -> { - th.size(maxURLsPerBucket).explain(false); - for (String bsf : bucketSortField) { - th.sort( - s -> - s.field( - fs -> - fs.field(bsf) - .order( - SortOrder - .Asc))); - } - return th; - })); - - // Build the terms (partition) aggregation with top_hits sub-agg - Aggregation.Builder.ContainerBuilder partitionAggBuilder = - new Aggregation.Builder() - .terms( - t -> { - t.field(partitionField).size(maxBucketNum); - // sort between buckets by the min sub-aggregation - if (StringUtils.isNotBlank(totalSortField)) { - t.order( - Collections.singletonList( - Collections.singletonMap( - "top_hit", SortOrder.Asc))); - } - return t; - }) - .aggregations("docs", topHitsAgg); - - // add the min sub-aggregation used for sorting between buckets - if (StringUtils.isNotBlank(totalSortField)) { - partitionAggBuilder.aggregations( - "top_hit", Aggregation.of(minAgg -> minAgg.min(m -> m.field(totalSortField)))); - } - - Aggregation partitionAgg = partitionAggBuilder.build(); - - // Build the search request - SearchRequest.Builder requestBuilder = - new SearchRequest.Builder() - .index(indexName) - .size(0) - .trackTotalHits(t -> t.enabled(false)) - .query( - q -> - q.bool( - b -> { - b.filter( - f -> - f.range( - r -> - r.field( - "nextFetchDate") - .lte( - JsonData - .of( - formattedQueryDate)))); - if (filterQueries != null) { - for (String fq : filterQueries) { - b.filter( - f -> - f.queryString( - qs -> - qs - .query( - fq))); - } - } - return b; - })); - - if (queryTimeout != -1) { - requestBuilder.timeout(queryTimeout + "s"); - } - - if (sample) { - // Wrap in a diversified sampler aggregation - requestBuilder.aggregations( - "sample", - Aggregation.of( - a -> - a.diversifiedSampler( - ds -> - ds.field(partitionField) - .maxDocsPerValue( - maxURLsPerBucket) - .shardSize( - maxURLsPerBucket - * maxBucketNum)) - .aggregations("partition", partitionAgg))); - } else { - requestBuilder.aggregations("partition", partitionAgg); - } - - // shard preference for routing - if (shardID != -1) { - requestBuilder.preference("_shards:" + shardID + "|_local"); - } - - SearchRequest request = requestBuilder.build(); - - // dump query to log - LOG.debug("{} OpenSearch query {}", logIdprefix, request); - - LOG.trace("{} isInQuery set to true", logIdprefix); - isInQuery.set(true); - - CompletableFuture.supplyAsync( - () -> { - try { - return client.search(request, JsonData.class); - } catch (IOException e) { - throw new CompletionException(e); - } - }, - queryExecutor) - .thenAccept(this::handleResponse) - .exceptionally( - e -> { - Throwable cause = e instanceof CompletionException ? e.getCause() : e; - LOG.error("{} Exception with OpenSearch query", logIdprefix, cause); - markQueryReceivedNow(); - return null; - }); - } - - /** - * Handles the search response from an asynchronous aggregation query, extracting URLs from term - * buckets and adding them to the buffer. - * - * @param response the search response containing aggregation results - */ - protected void handleResponse(SearchResponse response) { - long timeTaken = System.currentTimeMillis() - getTimeLastQuerySent(); - - Map aggregs = response.aggregations(); - - if (aggregs == null || aggregs.isEmpty()) { - markQueryReceivedNow(); - return; - } - - // Unwrap the sample aggregation if present - Aggregate sampleAgg = aggregs.get("sample"); - if (sampleAgg != null) { - aggregs = sampleAgg.sampler().aggregations(); - } - - Aggregate partitionAgg = aggregs.get("partition"); - List buckets = partitionAgg.sterms().buckets().array(); - - int numhits = 0; - int numBuckets = 0; - int alreadyprocessed = 0; - - Instant mostRecentDateFound = null; - - currentBuckets.clear(); - - // For each entry - Iterator iterator = buckets.iterator(); - while (iterator.hasNext()) { - StringTermsBucket entry = iterator.next(); - String key = entry.key(); // bucket key - - currentBuckets.add(key); - - long docCount = entry.docCount(); // Doc count - - int hitsForThisBucket = 0; - - List lastSortValues = null; - // filter results so that we don't include URLs we are already - // being processed - TopHitsAggregate topHits = entry.aggregations().get("docs").topHits(); - for (Hit hit : topHits.hits().hits()) { - - Map keyValues = sourceAsMap(hit.source()); - - LOG.debug("{} -> id [{}], _source [{}]", logIdprefix, hit.id(), keyValues); - - hitsForThisBucket++; - - lastSortValues = hit.sort(); - - String url = (String) keyValues.get("url"); - - // consider only the first document of the last bucket - // for optimising the nextFetchDate - if (hitsForThisBucket == 1 && !iterator.hasNext()) { - String strDate = (String) keyValues.get("nextFetchDate"); - try { - mostRecentDateFound = Instant.parse(strDate); - } catch (Exception e) { - throw new RuntimeException("can't parse date :" + strDate); - } - } - - // is already being processed or in buffer - skip it! - if (beingProcessed.containsKey(url)) { - LOG.debug("{} -> already processed: {}", logIdprefix, url); - alreadyprocessed++; - continue; - } - - Metadata metadata = fromKeyValues(keyValues); - boolean added = buffer.add(url, metadata); - if (!added) { - LOG.debug("{} -> already in buffer: {}", logIdprefix, url); - alreadyprocessed++; - continue; - } - LOG.debug("{} -> added to buffer : {}", logIdprefix, url); - } - - if (lastSortValues != null && !lastSortValues.isEmpty()) { - sortValuesForKey(key, lastSortValues.toArray()); - } - - if (hitsForThisBucket > 0) { - numBuckets++; - } - - numhits += hitsForThisBucket; - - LOG.debug( - "{} key [{}], hits[{}], doc_count [{}], already_processed [{}]", - logIdprefix, - key, - hitsForThisBucket, - docCount, - alreadyprocessed); - } - - LOG.info( - "{} OpenSearch query returned {} hits from {} buckets in {} msec with {} already being processed. Took {} msec per doc on average.", - logIdprefix, - numhits, - numBuckets, - timeTaken, - alreadyprocessed, - ((float) timeTaken / numhits)); - - queryTimes.accept(timeTaken); - eventCounter.scope("already_being_processed").incrBy(alreadyprocessed); - eventCounter.scope("OpenSearch_queries").incrBy(1); - eventCounter.scope("OpenSearch_docs").incrBy(numhits); - - // optimise the nextFetchDate by getting the most recent value - // returned in the query and add to it, unless the previous value is - // within n mins in which case we'll keep it - if (mostRecentDateFound != null && recentDateIncrease >= 0) { - Calendar potentialNewDate = - Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT); - potentialNewDate.setTimeInMillis(mostRecentDateFound.toEpochMilli()); - potentialNewDate.add(Calendar.MINUTE, recentDateIncrease); - Date oldDate = null; - // check boundaries - if (this.recentDateMinGap > 0) { - Calendar low = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT); - low.setTime(queryDate); - low.add(Calendar.MINUTE, -recentDateMinGap); - Calendar high = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT); - high.setTime(queryDate); - high.add(Calendar.MINUTE, recentDateMinGap); - if (high.before(potentialNewDate) || low.after(potentialNewDate)) { - oldDate = queryDate; - } - } else { - oldDate = queryDate; - } - if (oldDate != null) { - queryDate = potentialNewDate.getTime(); - LOG.info( - "{} queryDate changed from {} to {} based on mostRecentDateFound {}", - logIdprefix, - oldDate, - queryDate, - mostRecentDateFound); - } else { - LOG.info( - "{} queryDate kept at {} based on mostRecentDateFound {}", - logIdprefix, - queryDate, - mostRecentDateFound); - } - } - - // reset the value for next fetch date if the previous one is too old - if (resetFetchDateAfterNSecs != -1) { - Instant changeNeededOn = - Instant.ofEpochMilli( - lastTimeResetToNow.toEpochMilli() + (resetFetchDateAfterNSecs * 1000L)); - if (Instant.now().isAfter(changeNeededOn)) { - LOG.info( - "{} queryDate set to null based on resetFetchDateAfterNSecs {}", - logIdprefix, - resetFetchDateAfterNSecs); - queryDate = null; - } - } - - // change the date if we don't get any results at all - if (numBuckets == 0) { - queryDate = null; - } - - // remove lock - markQueryReceivedNow(); - } - - /** - * Receives the sort values of the last hit returned for a bucket, as the {@link FieldValue} - * instances of the client. Subclasses can pass them to {@code search_after} to read the bucket - * from that position. - */ - protected void sortValuesForKey(String key, Object[] sortValues) {} -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java deleted file mode 100644 index b3301e5c3..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.persistence; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import java.io.IOException; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.stormcrawler.opensearch.Constants; -import org.apache.stormcrawler.persistence.EmptyQueueListener; -import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.json.JsonData; -import org.opensearch.client.opensearch._types.FieldValue; -import org.opensearch.client.opensearch._types.SortOrder; -import org.opensearch.client.opensearch.core.SearchRequest; -import org.opensearch.client.opensearch.core.SearchResponse; -import org.opensearch.client.opensearch.core.search.Hit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Uses collapsing spouts to get an initial set of URLs and keys to query for and gets emptyQueue - * notifications from the URLBuffer to query OpenSearch for a specific key. - * - * @since 1.15 - */ -public class HybridSpout extends AggregationSpout implements EmptyQueueListener { - - private static final Logger LOG = LoggerFactory.getLogger(HybridSpout.class); - - protected static final String RELOADPARAMNAME = - Constants.PARAMPREFIX + "status.max.urls.per.reload"; - - private int bufferReloadSize = 10; - - private Cache searchAfterCache; - - private HostResultListener hrl; - - @Override - public void open( - Map stormConf, - TopologyContext context, - SpoutOutputCollector collector) { - super.open(stormConf, context, collector); - bufferReloadSize = ConfUtils.getInt(stormConf, RELOADPARAMNAME, maxURLsPerBucket); - buffer.setEmptyQueueListener(this); - searchAfterCache = Caffeine.newBuilder().build(); - hrl = new HostResultListener(); - } - - @Override - public void emptyQueue(String queueName) { - - LOG.info("{} Emptied buffer queue for {}", logIdprefix, queueName); - - if (!currentBuckets.contains(queueName)) { - // not interested in this one any more - return; - } - - // reloading the aggregs - searching now - // would just overload OpenSearch and yield - // mainly duplicates - if (isInQuery.get()) { - LOG.trace("{} isInquery true for {}", logIdprefix, queueName); - return; - } - - LOG.info("{} Querying for more docs for {}", logIdprefix, queueName); - - if (queryDate == null) { - queryDate = new Date(); - lastTimeResetToNow = Instant.now(); - } - - String formattedQueryDate = - Instant.ofEpochMilli(queryDate.getTime()) - .atOffset(ZoneOffset.UTC) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); - - SearchRequest.Builder requestBuilder = - new SearchRequest.Builder() - .index(indexName) - .size(bufferReloadSize) - .trackTotalHits(t -> t.enabled(false)) - .query( - q -> - q.bool( - b -> - b.filter( - f -> - f.range( - r -> - r.field( - "nextFetchDate") - .lte( - JsonData - .of( - formattedQueryDate)))) - .filter( - f -> - f.term( - t -> - t.field( - partitionField) - .value( - FieldValue - .of( - queueName)))))); - - // sort within a bucket - for (String bsf : bucketSortField) { - requestBuilder.sort(s -> s.field(fs -> fs.field(bsf).order(SortOrder.Asc))); - } - - // do we have a search after for this one? - Object[] searchAfterValues = searchAfterCache.getIfPresent(queueName); - if (searchAfterValues != null) { - for (Object sav : searchAfterValues) { - requestBuilder.searchAfter((FieldValue) sav); - } - } - - // shard preference for routing - if (shardID != -1) { - requestBuilder.preference("_shards:" + shardID + "|_local"); - } - - SearchRequest request = requestBuilder.build(); - - // dump query to log - LOG.debug("{} OpenSearch query {} - {}", logIdprefix, queueName, request); - - CompletableFuture.supplyAsync( - () -> { - try { - return client.search(request, JsonData.class); - } catch (IOException e) { - throw new CompletionException(e); - } - }, - queryExecutor) - .thenAccept(hrl::handleResponse) - .exceptionally( - e -> { - Throwable cause = e instanceof CompletionException ? e.getCause() : e; - LOG.error("Exception with OpenSearch query", cause); - return null; - }); - } - - /** Overrides the handling of responses for aggregations. */ - @Override - protected void handleResponse(SearchResponse response) { - // delete all entries from the searchAfterCache when - // we get the results from the aggregation spouts - searchAfterCache.invalidateAll(); - super.handleResponse(response); - } - - /** The aggregation kindly told us where to start from. */ - @Override - protected void sortValuesForKey(String key, Object[] sortValues) { - if (sortValues != null && sortValues.length > 0) { - this.searchAfterCache.put(key, sortValues); - } - } - - /** Handling of results for a specific queue. */ - class HostResultListener { - - /** - * Handles the search response for a host-specific query, extracting hits and adding them to - * the buffer. - * - * @param response the search response containing document hits - */ - void handleResponse(SearchResponse response) { - - int alreadyprocessed = 0; - int numDocs = 0; - - List> hits = response.hits().hits(); - - Object[] sortValues = null; - - // retrieve the key for these results - String key = null; - - for (Hit hit : hits) { - numDocs++; - - Map source = sourceAsMap(hit.source()); - - String pfield = partitionField; - Map fieldSource = source; - if (pfield.startsWith("metadata.")) { - @SuppressWarnings("unchecked") - Map metadataMap = (Map) source.get("metadata"); - fieldSource = metadataMap; - pfield = pfield.substring(9); - } - Object key_as_object = fieldSource.get(pfield); - if (key_as_object instanceof List) { - @SuppressWarnings("unchecked") - List keyList = (List) key_as_object; - if (keyList.size() == 1) { - key = keyList.get(0); - } - } else { - key = key_as_object.toString(); - } - - sortValues = hit.sort().toArray(); - if (!addHitToBuffer(source)) { - alreadyprocessed++; - } - } - - // no key if no results have been found - if (key != null) { - searchAfterCache.put(key, sortValues); - } - - eventCounter.scope("OpenSearch_queries_host").incrBy(1); - eventCounter.scope("OpenSearch_docs_host").incrBy(numDocs); - eventCounter.scope("already_being_processed_host").incrBy(alreadyprocessed); - - LOG.info( - "{} OpenSearch term query returned {} hits in {} msec with {} already being processed for {}", - logIdprefix, - numDocs, - response.took(), - alreadyprocessed, - key); - } - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java deleted file mode 100644 index 6af7d9094..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.persistence; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import java.io.IOException; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.tuple.Tuple; -import org.apache.stormcrawler.opensearch.AsyncBulkProcessor; -import org.apache.stormcrawler.opensearch.Constants; -import org.apache.stormcrawler.opensearch.IndexCreation; -import org.apache.stormcrawler.opensearch.OpenSearchConnection; -import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.opensearch.core.BulkRequest; -import org.opensearch.client.opensearch.core.BulkResponse; -import org.opensearch.client.opensearch.core.bulk.BulkOperation; -import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Sends information about the queues into an OpenSearch index. This has to be connected to a status - * updater bolt. - */ -public class QueueBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener { - - private static final Logger LOG = LoggerFactory.getLogger(QueueBolt.class); - - private static final String OSBoltType = "queues"; - - static final String OSQueuesIndexNameParamName = - Constants.PARAMPREFIX + OSBoltType + ".index.name"; - - /** - * Parameter name to configure the cache of known queues. @see - * https://github.com/ben-manes/caffeine/wiki/Specification Default value is - * "maximumSize=10000". - */ - static final String OSQueuesCacheSpecParamName = - Constants.PARAMPREFIX + OSBoltType + ".cache.spec"; - - private OutputCollector _collector; - - private String indexName; - - private OpenSearchConnection connection; - - /** Keys confirmed to be present in the index; populated on bulk success only */ - private Cache knownQueue; - - /** docID to key for the requests currently in the bulk processor */ - private final Map inFlight = new ConcurrentHashMap<>(); - - public QueueBolt() {} - - /** Sets the index name instead of taking it from the configuration. * */ - public QueueBolt(String indexName) { - this.indexName = indexName; - } - - @Override - public void prepare( - Map conf, TopologyContext context, OutputCollector collector) { - _collector = collector; - if (indexName == null) { - indexName = ConfUtils.getString(conf, QueueBolt.OSQueuesIndexNameParamName, OSBoltType); - } - try { - connection = OpenSearchConnection.getConnection(conf, OSBoltType, this); - } catch (Exception e1) { - LOG.error("Can't connect to OpenSearch", e1); - throw new RuntimeException(e1); - } - try { - IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG); - } catch (IOException e) { - throw new RuntimeException(e); - } - - final String cacheSpec = - ConfUtils.getString( - conf, QueueBolt.OSQueuesCacheSpecParamName, "maximumSize=10000"); - knownQueue = Caffeine.from(cacheSpec).build(); - } - - @Override - public void cleanup() { - if (connection != null) { - connection.close(); - } - } - - @Override - public void execute(Tuple tuple) { - - final String key = tuple.getStringByField("key"); - - // ack no matter what - the queue info is best effort bookkeeping - // and failed writes are retried when the key is next seen - _collector.ack(tuple); - - // check whether this key is already known - if (knownQueue.getIfPresent(key) != null) { - return; - } - - final String docID = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key); - - // a request for this key is already in the bulk processor - if (inFlight.putIfAbsent(docID, key) != null) { - return; - } - - final HashMap fields = new HashMap<>(); - fields.put("key", key); - fields.put("lastUpdated", Instant.now().toString()); - - final BulkOperation op = - BulkOperation.of(b -> b.create(c -> c.index(indexName).id(docID).document(fields))); - - connection.addToProcessor(op); - } - - @Override - public void beforeBulk(long executionId, BulkRequest request) { - LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size()); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - LOG.debug("afterBulk [{}] with {} responses", executionId, response.items().size()); - for (BulkResponseItem item : response.items()) { - final String key = inFlight.remove(item.id()); - if (key == null) { - continue; - } - if (item.error() == null) { - knownQueue.put(key, Boolean.TRUE); - } else if (item.status() == 409) { - // entry already exists in the index, e.g. written before a worker - // restart or evicted from the cache - not an error - knownQueue.put(key, Boolean.TRUE); - } else { - // do not cache the key so that the write is retried - // when the key is next seen - final var error = item.error(); - LOG.error( - "Failed to index queue entry {}: {}", - key, - error.reason() != null ? error.reason() : error.type()); - } - } - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - LOG.error("Exception with bulk {} - failing the whole lot ", executionId, failure); - for (BulkOperation op : request.operations()) { - // not cached - the writes are retried when the keys are next seen - inFlight.remove(OpenSearchConnection.getBulkOperationId(op)); - } - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - // nothing to do here - this bolt is the last of a topology - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java deleted file mode 100644 index 2e0b92eb8..000000000 --- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.persistence; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import org.apache.commons.lang3.StringUtils; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.metrics.CrawlerMetrics; -import org.apache.stormcrawler.metrics.ScopedCounter; -import org.apache.stormcrawler.metrics.ScopedReducedMetric; -import org.apache.stormcrawler.opensearch.AsyncBulkProcessor; -import org.apache.stormcrawler.opensearch.Constants; -import org.apache.stormcrawler.opensearch.IndexCreation; -import org.apache.stormcrawler.opensearch.OpenSearchConnection; -import org.apache.stormcrawler.opensearch.WaitAckCache; -import org.apache.stormcrawler.persistence.AbstractStatusUpdaterBolt; -import org.apache.stormcrawler.persistence.Status; -import org.apache.stormcrawler.util.ConfUtils; -import org.apache.stormcrawler.util.URLPartitioner; -import org.opensearch.client.opensearch.core.BulkRequest; -import org.opensearch.client.opensearch.core.BulkResponse; -import org.opensearch.client.opensearch.core.bulk.BulkOperation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Simple bolt which stores the status of URLs into OpenSearch. Takes the tuples coming from the - * 'status' stream. To be used in combination with a Spout to read from the index. - */ -public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt - implements AsyncBulkProcessor.Listener { - - private static final Logger LOG = LoggerFactory.getLogger(StatusUpdaterBolt.class); - - private String OSBoltType = "status"; - - private static final String OSStatusIndexNameParamName = - Constants.PARAMPREFIX + "%s.index.name"; - private static final String OSStatusRoutingParamName = Constants.PARAMPREFIX + "%s.routing"; - private static final String OSStatusRoutingFieldParamName = - Constants.PARAMPREFIX + "%s.routing.fieldname"; - - private boolean routingFieldNameInMetadata = false; - - private String indexName; - - private URLPartitioner partitioner; - - /** whether to apply the same partitioning logic used for politeness for routing, e.g byHost */ - private boolean doRouting; - - /** Store the key used for routing explicitly as a field in metadata * */ - private String fieldNameForRoutingKey = null; - - private OpenSearchConnection connection; - - private WaitAckCache waitAck; - - private ScopedCounter eventCounter; - - private ScopedReducedMetric receivedPerSecMetrics; - - public StatusUpdaterBolt() { - super(); - } - - /** - * Loads the configuration using a substring different from the default value 'status' in order - * to distinguish it from the spout configurations - */ - public StatusUpdaterBolt(String boltType) { - super(); - OSBoltType = boltType; - } - - @Override - public void prepare( - Map stormConf, TopologyContext context, OutputCollector collector) { - - super.prepare(stormConf, context, collector); - - indexName = - ConfUtils.getString( - stormConf, - String.format( - Locale.ROOT, - StatusUpdaterBolt.OSStatusIndexNameParamName, - OSBoltType), - "status"); - - doRouting = - ConfUtils.getBoolean( - stormConf, - String.format( - Locale.ROOT, - StatusUpdaterBolt.OSStatusRoutingParamName, - OSBoltType), - false); - - partitioner = new URLPartitioner(); - partitioner.configure(stormConf); - - fieldNameForRoutingKey = - ConfUtils.getString( - stormConf, - String.format( - Locale.ROOT, - StatusUpdaterBolt.OSStatusRoutingFieldParamName, - OSBoltType)); - if (StringUtils.isNotBlank(fieldNameForRoutingKey)) { - if (fieldNameForRoutingKey.startsWith("metadata.")) { - routingFieldNameInMetadata = true; - fieldNameForRoutingKey = fieldNameForRoutingKey.substring("metadata.".length()); - } - // periods are not allowed in - replace with %2E - fieldNameForRoutingKey = fieldNameForRoutingKey.replaceAll("\\.", "%2E"); - } - - int metrics_time_bucket_secs = 30; - - // benchmarking - average number of items received back from OpenSearch per second - this.receivedPerSecMetrics = - CrawlerMetrics.registerPerSecMetric( - context, stormConf, "average_persec", metrics_time_bucket_secs); - - // eventCounter MUST be registered before WaitAckCache — the eviction lambda captures it - this.eventCounter = - CrawlerMetrics.registerCounter( - context, stormConf, "counters", metrics_time_bucket_secs); - - String defaultSpec = - String.format( - Locale.ROOT, - "expireAfterWrite=%ds", - ConfUtils.getInt(stormConf, "topology.message.timeout.secs", 300)); - - String waitAckSpec = - ConfUtils.getString(stormConf, "opensearch.status.waitack.cache.spec", defaultSpec); - - waitAck = - new WaitAckCache( - waitAckSpec, - LOG, - t -> { - eventCounter.scope("purged").incrBy(1); - collector.fail(t); - }); - CrawlerMetrics.registerGauge( - context, stormConf, "waitAck", waitAck::estimatedSize, metrics_time_bucket_secs); - - try { - connection = OpenSearchConnection.getConnection(stormConf, OSBoltType, this); - } catch (Exception e1) { - LOG.error("Can't connect to OpenSearch", e1); - throw new RuntimeException(e1); - } - - // use the default status schema if none has been specified - try { - IndexCreation.checkOrCreateIndex(connection.getClient(), indexName, OSBoltType, LOG); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void cleanup() { - waitAck.shutdown(); - if (connection == null) { - return; - } - connection.close(); - connection = null; - } - - @Override - public void store( - String url, Status status, Metadata metadata, Optional nextFetch, Tuple tuple) - throws Exception { - - String documentID = getDocumentID(metadata, url); - - boolean isAlreadySentAndDiscovered = - status.equals(Status.DISCOVERED) && waitAck.contains(documentID); - - if (isAlreadySentAndDiscovered) { - // if this object is discovered - adding another version of it - // won't make any difference - LOG.debug( - "Already being sent to OpenSearch {} with status {} and ID {}", - url, - status, - documentID); - // ack straight away! - eventCounter.scope("skipped").incrBy(1); - super.ack(tuple, url); - return; - } - - Map doc = new HashMap<>(); - doc.put("url", url); - doc.put("status", status.name()); - - Map metadataMap = new HashMap<>(); - for (String mdKey : metadata.keySet()) { - String[] values = metadata.getValues(mdKey); - // periods are not allowed - replace with %2E - mdKey = mdKey.replaceAll("\\.", "%2E"); - metadataMap.put(mdKey, Arrays.asList(values)); - } - - String partitionKey = partitioner.getPartition(url, metadata); - if (partitionKey == null) { - partitionKey = "_DEFAULT_"; - } - - // send a tuple on the queue stream in case a bolt - // wants to handle it - super.collector.emit( - org.apache.stormcrawler.Constants.QUEUE_STREAM_NAME, - new Values(partitionKey, metadata)); - - // store routing key in metadata? - if (StringUtils.isNotBlank(fieldNameForRoutingKey) && routingFieldNameInMetadata) { - metadataMap.put(fieldNameForRoutingKey, partitionKey); - } - - doc.put("metadata", metadataMap); - - // store routing key outside metadata? - if (StringUtils.isNotBlank(fieldNameForRoutingKey) && !routingFieldNameInMetadata) { - doc.put(fieldNameForRoutingKey, partitionKey); - } - - if (nextFetch.isPresent()) { - doc.put("nextFetchDate", nextFetch.get().toInstant().toString()); - } - // check that we don't overwrite an existing entry - // When create is used, the index operation will fail if a document - // by that id already exists in the index. - final boolean create = status.equals(Status.DISCOVERED); - final String targetIndex = getIndexName(metadata); - final String routing = doRouting ? partitionKey : null; - - BulkOperation op; - if (create) { - op = - BulkOperation.of( - b -> - b.create( - c -> { - c.index(targetIndex).id(documentID).document(doc); - if (routing != null) { - c.routing(routing); - } - return c; - })); - } else { - op = - BulkOperation.of( - b -> - b.index( - idx -> { - idx.index(targetIndex).id(documentID).document(doc); - if (routing != null) { - idx.routing(routing); - } - return idx; - })); - } - - waitAck.addTuple(documentID, tuple); - - LOG.debug("Sending to OpenSearch buffer {} with ID {}", url, documentID); - - try { - connection.addToProcessor(op); - } catch (Exception e) { - // the caller fails the tuple, so no bulk response may ack or fail it again - waitAck.invalidate(documentID); - throw e; - } - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - LOG.debug("afterBulk [{}] with {} responses", executionId, request.operations().size()); - eventCounter.scope("bulks_received").incrBy(1); - eventCounter.scope("bulk_msec").incrBy(response.took()); - eventCounter.scope("received").incrBy(request.operations().size()); - receivedPerSecMetrics.scope("received").update(request.operations().size()); - - waitAck.processBulkResponse( - response, - executionId, - eventCounter, - (id, tuple, selected) -> { - if (!selected.failed()) { - String url = tuple.getStringByField("url"); - LOG.debug("Acked {} with ID {}", url, id); - eventCounter.scope("acked").incrBy(1); - super.ack(tuple, url); - } else { - eventCounter.scope("failed").incrBy(1); - collector.fail(tuple); - } - }); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable throwable) { - eventCounter.scope("bulks_received").incrBy(1); - eventCounter.scope("received").incrBy(request.operations().size()); - receivedPerSecMetrics.scope("received").update(request.operations().size()); - - waitAck.processFailedBulk( - request, - executionId, - throwable, - t -> { - eventCounter.scope("failed").incrBy(1); - collector.fail(t); - }); - } - - @Override - public void beforeBulk(long executionId, BulkRequest request) { - LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size()); - eventCounter.scope("bulks_sent").incrBy(1); - } - - /** - * Must be overridden for implementing custom index names based on some metadata information By - * Default, indexName coming from config is used - */ - protected String getIndexName(Metadata m) { - return indexName; - } -} diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java deleted file mode 100644 index 67bcca62b..000000000 --- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.stormcrawler.JSONResource; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.filtering.URLFilter; -import org.apache.stormcrawler.parse.ParseFilter; -import org.apache.stormcrawler.parse.ParseResult; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.jupiter.api.Test; -import org.w3c.dom.DocumentFragment; - -class DelegateRefresherTest { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - /** Minimal URLFilter + JSONResource implementation for testing. */ - public static class StubURLFilter extends URLFilter implements JSONResource { - - public final AtomicBoolean configured = new AtomicBoolean(false); - - @Override - public void configure(@NotNull Map stormConf, @NotNull JsonNode params) { - configured.set(true); - } - - @Override - public @Nullable String filter( - @Nullable URL sourceUrl, - @Nullable Metadata sourceMetadata, - @NotNull String urlToFilter) { - return urlToFilter; - } - - @Override - public String getResourceFile() { - return "stub.json"; - } - - @Override - public void loadJSONResources(InputStream inputStream) throws IOException {} - } - - /** Minimal ParseFilter + JSONResource implementation for testing. */ - public static class StubParseFilter extends ParseFilter implements JSONResource { - - public final AtomicBoolean configured = new AtomicBoolean(false); - - @Override - public void configure(@NotNull Map stormConf, @NotNull JsonNode params) { - configured.set(true); - } - - @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {} - - @Override - public String getResourceFile() { - return "stub.json"; - } - - @Override - public void loadJSONResources(InputStream inputStream) throws IOException {} - } - - /** A URLFilter that does NOT implement JSONResource. */ - public static class NonJsonResourceURLFilter extends URLFilter { - - @Override - public @Nullable String filter( - @Nullable URL sourceUrl, - @Nullable Metadata sourceMetadata, - @NotNull String urlToFilter) { - return urlToFilter; - } - } - - /** Not a URLFilter at all. */ - public static class NotAFilter {} - - private JsonNode buildParams(String delegateClass) { - return buildParams(delegateClass, 600); - } - - private JsonNode buildParams(String delegateClass, int refreshRate) { - ObjectNode delegate = MAPPER.createObjectNode(); - delegate.put("class", delegateClass); - delegate.set("params", MAPPER.createObjectNode()); - - ObjectNode params = MAPPER.createObjectNode(); - params.set("delegate", delegate); - params.put("refresh", refreshRate); - return params; - } - - @Test - void loadsURLFilterDelegate() { - JsonNode params = buildParams(StubURLFilter.class.getName()); - Map conf = new HashMap<>(); - - DelegateRefresher refresher = - new DelegateRefresher<>( - URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p)); - - try { - assertNotNull(refresher.getDelegate()); - assertInstanceOf(StubURLFilter.class, refresher.getDelegate()); - assertTrue(((StubURLFilter) refresher.getDelegate()).configured.get()); - } finally { - refresher.cleanup(); - } - } - - @Test - void loadsParseFilterDelegate() { - JsonNode params = buildParams(StubParseFilter.class.getName()); - Map conf = new HashMap<>(); - - DelegateRefresher refresher = - new DelegateRefresher<>( - ParseFilter.class, conf, params, (d, c, p) -> d.configure(c, p)); - - try { - assertNotNull(refresher.getDelegate()); - assertInstanceOf(StubParseFilter.class, refresher.getDelegate()); - assertTrue(((StubParseFilter) refresher.getDelegate()).configured.get()); - } finally { - refresher.cleanup(); - } - } - - @Test - void delegateFilterActuallyWorks() { - JsonNode params = buildParams(StubURLFilter.class.getName()); - Map conf = new HashMap<>(); - - DelegateRefresher refresher = - new DelegateRefresher<>( - URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p)); - - try { - String result = refresher.getDelegate().filter(null, null, "http://example.com"); - assertEquals("http://example.com", result); - } finally { - refresher.cleanup(); - } - } - - @Test - void throwsWhenDelegateNodeMissing() { - ObjectNode params = MAPPER.createObjectNode(); - // no "delegate" key - Map conf = new HashMap<>(); - - assertThrows( - RuntimeException.class, - () -> - new DelegateRefresher<>( - URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p))); - } - - @Test - void throwsWhenClassMissing() { - ObjectNode delegate = MAPPER.createObjectNode(); - // no "class" key - ObjectNode params = MAPPER.createObjectNode(); - params.set("delegate", delegate); - Map conf = new HashMap<>(); - - assertThrows( - RuntimeException.class, - () -> - new DelegateRefresher<>( - URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p))); - } - - @Test - void throwsWhenClassDoesNotExtendBaseType() { - JsonNode params = buildParams(NotAFilter.class.getName()); - Map conf = new HashMap<>(); - - RuntimeException ex = - assertThrows( - RuntimeException.class, - () -> - new DelegateRefresher<>( - URLFilter.class, - conf, - params, - (d, c, p) -> d.configure(c, p))); - assertTrue(ex.getMessage().contains("does not extend")); - } - - @Test - void throwsWhenClassDoesNotImplementJSONResource() { - JsonNode params = buildParams(NonJsonResourceURLFilter.class.getName()); - Map conf = new HashMap<>(); - - RuntimeException ex = - assertThrows( - RuntimeException.class, - () -> - new DelegateRefresher<>( - URLFilter.class, - conf, - params, - (d, c, p) -> d.configure(c, p))); - assertTrue(ex.getMessage().contains("does not implement JSONResource")); - } - - @Test - void cleanupIsIdempotent() { - JsonNode params = buildParams(StubURLFilter.class.getName()); - Map conf = new HashMap<>(); - - DelegateRefresher refresher = - new DelegateRefresher<>( - URLFilter.class, conf, params, (d, c, p) -> d.configure(c, p)); - - // calling cleanup twice should not throw - refresher.cleanup(); - refresher.cleanup(); - } -} diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java deleted file mode 100644 index bb484e843..000000000 --- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import org.apache.hc.client5.http.auth.AuthScope; -import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; -import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; -import org.apache.hc.core5.http.HttpHost; -import org.junit.jupiter.api.Test; - -class OpenSearchConnectionCredentialsTest { - - private static BasicCredentialsProvider providerFor(List hosts) { - final BasicCredentialsProvider provider = new BasicCredentialsProvider(); - final UsernamePasswordCredentials credentials = - new UsernamePasswordCredentials("crawler", "s3cret".toCharArray()); - for (AuthScope scope : OpenSearchConnection.credentialScopes(hosts)) { - provider.setCredentials(scope, credentials); - } - return provider; - } - - @Test - void scopesCoverTheConfiguredHostsAndPorts() { - final List scopes = - OpenSearchConnection.credentialScopes( - List.of( - new HttpHost("https", "opensearch1.example.org", 9200), - new HttpHost("https", "opensearch2.example.org", 9201))); - assertEquals( - List.of( - new AuthScope("https", "opensearch1.example.org", 9200, null, null), - new AuthScope("https", "opensearch2.example.org", 9201, null, null)), - scopes); - } - - @Test - void duplicateAddressesGiveOneScope() { - final List scopes = - OpenSearchConnection.credentialScopes( - List.of( - new HttpHost("https", "opensearch1.example.org", 9200), - new HttpHost("https", "OpenSearch1.example.org", 9200))); - assertEquals(1, scopes.size()); - } - - @Test - void noAddressGivesNoScope() { - assertTrue(OpenSearchConnection.credentialScopes(List.of()).isEmpty()); - } - - @Test - void credentialsAreOnlyGivenToTheConfiguredHosts() { - final BasicCredentialsProvider provider = - providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200))); - - assertNotNull( - provider.getCredentials(request("https", "opensearch1.example.org", 9200), null)); - // another node of the cluster which is not listed in the addresses - assertNull(provider.getCredentials(request("https", "10.0.0.12", 9200), null)); - // same host, another port - assertNull( - provider.getCredentials(request("https", "opensearch1.example.org", 9300), null)); - assertNull(provider.getCredentials(request("https", "other.example.org", 9200), null)); - } - - @Test - void credentialsAreMatchedOnTheScheme() { - final BasicCredentialsProvider provider = - providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200))); - assertNull(provider.getCredentials(request("http", "opensearch1.example.org", 9200), null)); - } - - /** The scope HttpClient looks the credentials up with for a request to the given node. */ - private static AuthScope request(String scheme, String host, int port) { - return new AuthScope(new HttpHost(scheme, host, port), null, "Basic"); - } - - @Test - void plainHttpToARemoteHostIsReported() { - final HttpHost remote = new HttpHost("http", "opensearch1.example.org", 9200); - final List plain = - OpenSearchConnection.plainHttpHosts( - List.of( - remote, - new HttpHost("https", "opensearch2.example.org", 9200), - new HttpHost("http", "localhost", 9200), - new HttpHost("http", "127.0.0.1", 9200), - new HttpHost("http", "[::1]", 9200))); - assertEquals(List.of(remote), plain); - } - - @Test - void loopbackAddresses() { - assertTrue(OpenSearchConnection.isLoopback("localhost")); - assertTrue(OpenSearchConnection.isLoopback("LOCALHOST")); - assertTrue(OpenSearchConnection.isLoopback("127.0.0.1")); - assertTrue(OpenSearchConnection.isLoopback("[::1]")); - assertTrue(OpenSearchConnection.isLoopback("0:0:0:0:0:0:0:1")); - assertFalse(OpenSearchConnection.isLoopback("opensearch1.example.org")); - assertFalse(OpenSearchConnection.isLoopback("10.0.0.12")); - assertFalse(OpenSearchConnection.isLoopback("localhost.example.org")); - assertFalse(OpenSearchConnection.isLoopback("127.example.org")); - assertTrue(OpenSearchConnection.isLoopback("127.1.2.3")); - } -} diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java deleted file mode 100644 index 3bd2dc6d4..000000000 --- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch; - -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; -import org.apache.storm.tuple.Tuple; -import org.apache.stormcrawler.metrics.ScopedCounter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.opensearch.client.opensearch._types.ErrorCause; -import org.opensearch.client.opensearch.core.BulkRequest; -import org.opensearch.client.opensearch.core.BulkResponse; -import org.opensearch.client.opensearch.core.bulk.BulkOperation; -import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; -import org.opensearch.client.opensearch.core.bulk.OperationType; -import org.slf4j.LoggerFactory; - -class WaitAckCacheTest { - - private WaitAckCache cache; - private List evicted; - private List acked; - private List failed; - - @BeforeEach - void setUp() { - evicted = new CopyOnWriteArrayList<>(); - acked = new ArrayList<>(); - failed = new ArrayList<>(); - cache = new WaitAckCache(LoggerFactory.getLogger(WaitAckCacheTest.class), evicted::add); - } - - private Tuple mockTuple(String url) { - Tuple t = mock(Tuple.class); - when(t.getValueByField("url")).thenReturn(url); - when(t.getStringByField("url")).thenReturn(url); - return t; - } - - private static BulkResponseItem successItem(String docId) { - return BulkResponseItem.of( - b -> b.id(docId).index("index").status(200).operationType(OperationType.Index)); - } - - private static BulkResponseItem failedItem(String docId, int status) { - return BulkResponseItem.of( - b -> - b.id(docId) - .index("index") - .status(status) - .operationType(OperationType.Index) - .error( - ErrorCause.of( - e -> e.type("test_error").reason("test failure")))); - } - - private static BulkResponse bulkResponse(BulkResponseItem... items) { - boolean hasErrors = false; - for (BulkResponseItem item : items) { - if (item.error() != null) { - hasErrors = true; - break; - } - } - final boolean errors = hasErrors; - return BulkResponse.of(b -> b.took(10).errors(errors).items(List.of(items))); - } - - @Test - void addAndContains() { - Tuple t = mockTuple("http://example.com"); - assertFalse(cache.contains("doc1")); - - cache.addTuple("doc1", t); - assertTrue(cache.contains("doc1")); - assertEquals(1, cache.estimatedSize()); - } - - @Test - void invalidateRemovesEntry() { - Tuple t = mockTuple("http://example.com"); - cache.addTuple("doc1", t); - assertTrue(cache.contains("doc1")); - - cache.invalidate("doc1"); - assertFalse(cache.contains("doc1")); - } - - @Test - void processBulkResponse_successfulItem_ackedViaTupleAction() { - Tuple t = mockTuple("http://example.com"); - cache.addTuple("doc1", t); - - BulkResponse response = bulkResponse(successItem("doc1")); - - cache.processBulkResponse( - response, - 1L, - null, - (id, tuple, selected) -> { - if (!selected.failed()) { - acked.add(tuple); - } else { - failed.add(tuple); - } - }); - - assertEquals(1, acked.size()); - assertEquals(0, failed.size()); - assertSame(t, acked.get(0)); - assertFalse(cache.contains("doc1")); - } - - @Test - void processBulkResponse_failedItem_failedViaTupleAction() { - Tuple t = mockTuple("http://example.com"); - cache.addTuple("doc1", t); - - BulkResponse response = bulkResponse(failedItem("doc1", 500)); - - cache.processBulkResponse( - response, - 1L, - null, - (id, tuple, selected) -> { - if (!selected.failed()) { - acked.add(tuple); - } else { - failed.add(tuple); - } - }); - - assertEquals(0, acked.size()); - assertEquals(1, failed.size()); - assertSame(t, failed.get(0)); - } - - @Test - void processBulkResponse_conflictIsNotAFailure() { - Tuple t = mockTuple("http://example.com"); - cache.addTuple("doc1", t); - - ScopedCounter counter = mock(ScopedCounter.class); - ScopedCounter.CountHandle handle = mock(ScopedCounter.CountHandle.class); - when(counter.scope("doc_conflicts")).thenReturn(handle); - - BulkResponse response = bulkResponse(failedItem("doc1", 409)); - - cache.processBulkResponse( - response, - 1L, - counter, - (id, tuple, selected) -> { - if (!selected.failed()) { - acked.add(tuple); - } else { - failed.add(tuple); - } - }); - - assertEquals(1, acked.size()); - assertEquals(0, failed.size()); - verify(handle).incrBy(1); - } - - @Test - void processBulkResponse_multipleTuplesForSameDocId() { - Tuple t1 = mockTuple("http://example.com/1"); - Tuple t2 = mockTuple("http://example.com/2"); - cache.addTuple("doc1", t1); - cache.addTuple("doc1", t2); - - BulkResponse response = bulkResponse(successItem("doc1")); - - cache.processBulkResponse(response, 1L, null, (id, tuple, selected) -> acked.add(tuple)); - - assertEquals(2, acked.size()); - assertTrue(acked.contains(t1)); - assertTrue(acked.contains(t2)); - } - - @Test - void processBulkResponse_duplicateDocIdInBulk_prefersSuccess() { - // https://github.com/apache/stormcrawler/issues/832 - Tuple t = mockTuple("http://example.com"); - cache.addTuple("doc1", t); - - BulkResponse response = bulkResponse(failedItem("doc1", 500), successItem("doc1")); - - cache.processBulkResponse( - response, - 1L, - null, - (id, tuple, selected) -> { - if (!selected.failed()) { - acked.add(tuple); - } else { - failed.add(tuple); - } - }); - - assertEquals(1, acked.size()); - assertEquals(0, failed.size()); - } - - @Test - void processFailedBulk_failsAllMatchingTuples() { - Tuple t1 = mockTuple("http://example.com/1"); - Tuple t2 = mockTuple("http://example.com/2"); - cache.addTuple("doc1", t1); - cache.addTuple("doc2", t2); - - BulkRequest request = - BulkRequest.of( - b -> - b.operations( - BulkOperation.of( - o -> o.delete(d -> d.index("index").id("doc1"))), - BulkOperation.of( - o -> o.delete(d -> d.index("index").id("doc2"))))); - - cache.processFailedBulk(request, 1L, new Exception("connection lost"), failed::add); - - assertEquals(2, failed.size()); - assertTrue(failed.contains(t1)); - assertTrue(failed.contains(t2)); - assertFalse(cache.contains("doc1")); - assertFalse(cache.contains("doc2")); - } - - @Test - void processFailedBulk_ignoresMissingIds() { - Tuple t = mockTuple("http://example.com"); - cache.addTuple("doc1", t); - - BulkRequest request = - BulkRequest.of( - b -> - b.operations( - BulkOperation.of( - o -> - o.delete( - d -> - d.index("index") - .id( - "doc_unknown"))))); - - cache.processFailedBulk(request, 1L, new Exception("test"), failed::add); - - assertEquals(0, failed.size()); - // doc1 should still be in cache since it wasn't in the failed request - assertTrue(cache.contains("doc1")); - } - - @Test - void eviction_failsTuplesOnExpiry() { - cache = - new WaitAckCache( - "expireAfterWrite=1s", - LoggerFactory.getLogger(WaitAckCacheTest.class), - evicted::add); - Tuple t = mockTuple("http://example.com"); - cache.addTuple("doc1", t); - - // Force cache maintenance after expiry by doing a contains() check - // which accesses the cache and triggers Caffeine's cleanup - await().atMost(5, TimeUnit.SECONDS) - .pollInterval(200, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> { - // contains() accesses the cache which triggers cleanup - cache.contains("doc1"); - // also try adding and invalidating a dummy entry to force maintenance - Tuple dummy = mockTuple("http://dummy"); - cache.addTuple("_probe_", dummy); - cache.invalidate("_probe_"); - assertFalse(evicted.isEmpty(), "Eviction callback should have fired"); - }); - - assertTrue(evicted.contains(t)); - } - - @Test - void processBulkResponse_multipleDocIds() { - Tuple t1 = mockTuple("http://example.com/1"); - Tuple t2 = mockTuple("http://example.com/2"); - cache.addTuple("doc1", t1); - cache.addTuple("doc2", t2); - - BulkResponse response = bulkResponse(successItem("doc1"), failedItem("doc2", 500)); - - cache.processBulkResponse( - response, - 1L, - null, - (id, tuple, selected) -> { - if (!selected.failed()) { - acked.add(tuple); - } else { - failed.add(tuple); - } - }); - - assertEquals(1, acked.size()); - assertSame(t1, acked.get(0)); - assertEquals(1, failed.size()); - assertSame(t2, failed.get(0)); - } -} diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java deleted file mode 100644 index 44150ea67..000000000 --- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.bolt; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.utility.DockerImageName; - -@Testcontainers(disabledWithoutDocker = true) -public abstract class AbstractOpenSearchTest { - - private static final String OPENSEARCH_VERSION = - System.getProperty("opensearch-version", "2.19.5"); - - public static final String PASSWORD = "This1sAPassw0rd"; - - protected GenericContainer opensearchContainer = - new GenericContainer<>( - DockerImageName.parse( - "opensearchproject/opensearch:" + OPENSEARCH_VERSION)) - .withExposedPorts(9200) - .withEnv("plugins.security.disabled", "true") - .withEnv("discovery.type", "single-node") - .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m") - .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", PASSWORD); - - @BeforeEach - void init() { - opensearchContainer.start(); - } - - @AfterEach - void close() { - opensearchContainer.close(); - } -} diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java deleted file mode 100644 index aa953a283..000000000 --- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/IndexerBoltTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.bolt; - -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.tuple.Tuple; -import org.apache.stormcrawler.Constants; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.TestOutputCollector; -import org.apache.stormcrawler.TestUtil; -import org.apache.stormcrawler.indexing.AbstractIndexerBolt; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class IndexerBoltTest extends AbstractOpenSearchTest { - - private IndexerBolt bolt; - - protected TestOutputCollector output; - - private static final Logger LOG = LoggerFactory.getLogger(IndexerBoltTest.class); - - private static ExecutorService executorService; - - @BeforeAll - static void beforeClass() { - executorService = Executors.newFixedThreadPool(2); - } - - @AfterAll - static void afterClass() { - executorService.shutdown(); - executorService = null; - } - - @BeforeEach - void setupIndexerBolt() { - bolt = new IndexerBolt("content"); - // give the indexer the port for connecting to OpenSearch - final String host = opensearchContainer.getHost(); - final Integer port = opensearchContainer.getFirstMappedPort(); - final Map conf = new HashMap<>(); - conf.put(AbstractIndexerBolt.urlFieldParamName, "url"); - conf.put(AbstractIndexerBolt.canonicalMetadataParamName, "canonical"); - conf.put("opensearch.indexer.addresses", host + ":" + port); - output = new TestOutputCollector(); - bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); - } - - @AfterEach - void close() { - LOG.info("Closing indexer bolt and Opensearch container"); - super.close(); - bolt.cleanup(); - output = null; - } - - private void index(String url, String text, Metadata metadata) { - Tuple tuple = mock(Tuple.class); - when(tuple.getStringByField("text")).thenReturn(text); - when(tuple.getStringByField("url")).thenReturn(url); - when(tuple.getValueByField("metadata")).thenReturn(metadata); - bolt.execute(tuple); - } - - private int lastIndex(String url, String text, Metadata metadata, long timeoutInMs) - throws ExecutionException, InterruptedException, TimeoutException { - var oldSize = output.getEmitted(Constants.StatusStreamName).size(); - index(url, text, metadata); - return executorService - .submit( - () -> { - await().atMost(timeoutInMs, TimeUnit.MILLISECONDS) - .until( - () -> - output.getEmitted(Constants.StatusStreamName) - .size() - > oldSize); - return output.getEmitted(Constants.StatusStreamName).size(); - }) - .get(timeoutInMs, TimeUnit.MILLISECONDS); - } - - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - // https://github.com/apache/stormcrawler/issues/832 - void simultaneousCanonicals() - throws ExecutionException, InterruptedException, TimeoutException { - Metadata m1 = new Metadata(); - String url = - "https://www.obozrevatel.com/ukr/dnipro/city/u-dnipri-ta-oblasti-ogolosili-shtormove-poperedzhennya.htm"; - m1.addValue("canonical", url); - Metadata m2 = new Metadata(); - String url2 = - "https://www.obozrevatel.com/ukr/dnipro/city/u-dnipri-ta-oblasti-ogolosili-shtormove-poperedzhennya/amp.htm"; - m2.addValue("canonical", url); - index(url, "", m1); - lastIndex(url2, "", m2, 10_000); - // should be two in status output - assertEquals(2, output.getEmitted(Constants.StatusStreamName).size()); - // and 2 acked - assertEquals(2, output.getAckedTuples().size()); - // TODO check output in Opensearch? - } -} diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java deleted file mode 100644 index 296c58dd5..000000000 --- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.bolt; - -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.hc.core5.http.HttpHost; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.tuple.Tuple; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.TestOutputCollector; -import org.apache.stormcrawler.TestUtil; -import org.apache.stormcrawler.opensearch.persistence.QueueBolt; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.opensearch.client.json.jackson.JacksonJsonpMapper; -import org.opensearch.client.opensearch.OpenSearchClient; -import org.opensearch.client.opensearch.core.GetResponse; -import org.opensearch.client.transport.OpenSearchTransport; -import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class QueueBoltTest extends AbstractOpenSearchTest { - - private QueueBolt bolt; - - protected TestOutputCollector output; - - protected OpenSearchClient client; - - private OpenSearchTransport transport; - - private static final Logger LOG = LoggerFactory.getLogger(QueueBoltTest.class); - - private static ExecutorService executorService; - - @BeforeAll - static void beforeClass() { - executorService = Executors.newFixedThreadPool(2); - } - - @AfterAll - static void afterClass() { - executorService.shutdown(); - executorService = null; - } - - @BeforeEach - void setupQueueBolt() throws IOException { - transport = - ApacheHttpClient5TransportBuilder.builder( - new HttpHost( - "http", - opensearchContainer.getHost(), - opensearchContainer.getMappedPort(9200))) - .setMapper(new JacksonJsonpMapper()) - .build(); - client = new OpenSearchClient(transport); - output = new TestOutputCollector(); - bolt = prepareQueueBolt(); - } - - private QueueBolt prepareQueueBolt() { - QueueBolt queueBolt = new QueueBolt(); - Map conf = new HashMap<>(); - conf.put( - "opensearch.queues.addresses", - opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort()); - queueBolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); - return queueBolt; - } - - @AfterEach - void close() { - LOG.info("Closing queue bolt and Opensearch container"); - super.close(); - bolt.cleanup(); - output = null; - try { - transport.close(); - } catch (IOException e) { - } - } - - private Future store(String key, Metadata metadata) { - Tuple tuple = mock(Tuple.class); - when(tuple.getStringByField("key")).thenReturn(key); - when(tuple.getValueByField("metadata")).thenReturn(metadata); - bolt.execute(tuple); - return executorService.submit( - () -> { - await().atMost(30, TimeUnit.SECONDS) - .until(() -> output.getAckedTuples().size() > 0); - return output.getAckedTuples().size(); - }); - } - - private void awaitIndexed(String key) { - String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key); - await().atMost(30, TimeUnit.SECONDS) - .until( - () -> { - try { - GetResponse result = - client.get(g -> g.index("queues").id(id), Map.class); - return result.found(); - } catch (IOException e) { - return false; - } - }); - } - - @SuppressWarnings("unchecked") - private Map getSource(String key) throws IOException { - String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key); - return client.get(g -> g.index("queues").id(id), Map.class).source(); - } - - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - void checkQueueIndex() - throws IOException, ExecutionException, InterruptedException, TimeoutException { - String key = "www.url.net"; - Metadata md = new Metadata(); - md.addValue("someKey", "someValue"); - store(key, md).get(10, TimeUnit.SECONDS); - assertEquals(1, output.getAckedTuples().size()); - - // Wait until document is indexed in OpenSearch - awaitIndexed(key); - - Map sourceAsMap = getSource(key); - assertEquals(key, sourceAsMap.get("key")); - } - - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - void checkExistingEntryIsNotOverwritten() - throws IOException, ExecutionException, InterruptedException, TimeoutException { - String key = "www.url.net"; - Metadata md = new Metadata(); - md.addValue("someKey", "someValue"); - store(key, md).get(10, TimeUnit.SECONDS); - awaitIndexed(key); - Object lastUpdated = getSource(key).get("lastUpdated"); - - // a fresh bolt instance (e.g. after a worker restart) has an empty cache - // and re-sends the same key - the resulting version conflict must be - // handled gracefully and the existing entry left untouched - QueueBolt restartedBolt = prepareQueueBolt(); - try { - Tuple sameKey = mock(Tuple.class); - when(sameKey.getStringByField("key")).thenReturn(key); - restartedBolt.execute(sameKey); - - // a new key sent in the same bulk signals when the bulk has been processed - String otherKey = "www.other.net"; - Tuple other = mock(Tuple.class); - when(other.getStringByField("key")).thenReturn(otherKey); - restartedBolt.execute(other); - awaitIndexed(otherKey); - - Map sourceAsMap = getSource(key); - assertEquals(key, sourceAsMap.get("key")); - assertEquals(lastUpdated, sourceAsMap.get("lastUpdated")); - } finally { - restartedBolt.cleanup(); - } - } -} diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java deleted file mode 100644 index 71fe77957..000000000 --- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.bolt; - -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.hc.core5.http.HttpHost; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.tuple.Tuple; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.TestOutputCollector; -import org.apache.stormcrawler.TestUtil; -import org.apache.stormcrawler.opensearch.persistence.StatusUpdaterBolt; -import org.apache.stormcrawler.persistence.Status; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.opensearch.client.json.jackson.JacksonJsonpMapper; -import org.opensearch.client.opensearch.OpenSearchClient; -import org.opensearch.client.opensearch.core.GetResponse; -import org.opensearch.client.transport.OpenSearchTransport; -import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class StatusBoltTest extends AbstractOpenSearchTest { - - private StatusUpdaterBolt bolt; - - protected TestOutputCollector output; - - protected OpenSearchClient client; - - private OpenSearchTransport transport; - - private static final Logger LOG = LoggerFactory.getLogger(StatusBoltTest.class); - - private static ExecutorService executorService; - - @BeforeAll - static void beforeClass() { - executorService = Executors.newFixedThreadPool(2); - } - - @AfterAll - static void afterClass() { - executorService.shutdown(); - executorService = null; - } - - @BeforeEach - void setupStatusBolt() throws IOException { - bolt = new StatusUpdaterBolt(); - transport = - ApacheHttpClient5TransportBuilder.builder( - new HttpHost( - "http", - opensearchContainer.getHost(), - opensearchContainer.getMappedPort(9200))) - .setMapper(new JacksonJsonpMapper()) - .build(); - client = new OpenSearchClient(transport); - // configure the status updater bolt - Map conf = new HashMap<>(); - conf.put("opensearch.status.routing.fieldname", "metadata.key"); - conf.put( - "opensearch.status.addresses", - opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort()); - conf.put("scheduler.class", "org.apache.stormcrawler.persistence.DefaultScheduler"); - conf.put("status.updater.cache.spec", "maximumSize=10000,expireAfterAccess=1h"); - conf.put("metadata.persist", "someKey"); - output = new TestOutputCollector(); - bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); - } - - @AfterEach - void close() { - LOG.info("Closing updater bolt and Opensearch container"); - super.close(); - bolt.cleanup(); - output = null; - try { - transport.close(); - } catch (IOException e) { - } - } - - private Future store(String url, Status status, Metadata metadata) { - Tuple tuple = mock(Tuple.class); - when(tuple.getValueByField("status")).thenReturn(status); - when(tuple.getStringByField("url")).thenReturn(url); - when(tuple.getValueByField("metadata")).thenReturn(metadata); - bolt.execute(tuple); - return executorService.submit( - () -> { - await().atMost(30, TimeUnit.SECONDS) - .until(() -> output.getAckedTuples().size() > 0); - return output.getAckedTuples().size(); - }); - } - - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - // see https://github.com/apache/stormcrawler/issues/885 - @SuppressWarnings("unchecked") - void checkListKeyFromOpensearch() - throws IOException, ExecutionException, InterruptedException, TimeoutException { - String url = "https://www.url.net/something"; - Metadata md = new Metadata(); - md.addValue("someKey", "someValue"); - store(url, Status.DISCOVERED, md).get(10, TimeUnit.SECONDS); - assertEquals(1, output.getAckedTuples().size()); - // check output in Opensearch - String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(url); - GetResponse result = client.get(g -> g.index("status").id(id), Map.class); - Map sourceAsMap = result.source(); - final String pfield = "metadata.somekey"; - sourceAsMap = (Map) sourceAsMap.get("metadata"); - final var pfieldNew = pfield.substring(9); - Object key = sourceAsMap.get(pfieldNew); - assertTrue(key instanceof java.util.ArrayList); - } - - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - void checkQueueStreamEmission() - throws IOException, ExecutionException, InterruptedException, TimeoutException { - String url = "https://www.url.net/something"; - Metadata md = new Metadata(); - md.addValue("someKey", "someValue"); - store(url, Status.DISCOVERED, md).get(10, TimeUnit.SECONDS); - assertEquals(1, output.getAckedTuples().size()); - - // Check that a tuple was emitted on the queue stream - var emittedQueue = output.getEmitted(org.apache.stormcrawler.Constants.QUEUE_STREAM_NAME); - assertEquals(1, emittedQueue.size()); - - var queueTuple = emittedQueue.get(0); - assertEquals(2, queueTuple.size()); - assertEquals("www.url.net", queueTuple.get(0)); - assertTrue(queueTuple.get(1) instanceof Metadata); - Metadata emittedMetadata = (Metadata) queueTuple.get(1); - assertEquals("someValue", emittedMetadata.getFirstValue("someKey")); - } -} diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java b/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java deleted file mode 100644 index 8c0c9413e..000000000 --- a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporterTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.metrics; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.MetricRegistry; -import java.util.HashMap; -import java.util.Map; -import org.apache.stormcrawler.opensearch.bolt.AbstractOpenSearchTest; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -class MetricsReporterTest extends AbstractOpenSearchTest { - - @Test - @Timeout(60) - void prepareAndReportMetrics() { - MetricRegistry registry = new MetricRegistry(); - Counter counter = registry.counter("test.counter"); - counter.inc(42); - - Map topoConf = new HashMap<>(); - topoConf.put( - "opensearch.metrics.addresses", - opensearchContainer.getHost() + ":" + opensearchContainer.getFirstMappedPort()); - - Map reporterConf = new HashMap<>(); - reporterConf.put("report.period", 60L); - reporterConf.put("report.period.units", "SECONDS"); - - MetricsReporter reporter = new MetricsReporter(); - assertDoesNotThrow(() -> reporter.prepare(registry, topoConf, reporterConf)); - assertNotNull(reporter); - reporter.stop(); - } -} diff --git a/external/opensearch-java/src/test/resources/indexer.mapping b/external/opensearch-java/src/test/resources/indexer.mapping deleted file mode 100644 index fc6eb887f..000000000 --- a/external/opensearch-java/src/test/resources/indexer.mapping +++ /dev/null @@ -1,40 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 5, - "number_of_replicas": 1, - "refresh_interval": "60s" - } - }, - "mappings": { - "_source": { - "enabled": true - }, - "properties": { - "content": { - "type": "text" - }, - "description": { - "type": "text" - }, - "domain": { - "type": "keyword" - }, - "format": { - "type": "keyword" - }, - "keywords": { - "type": "keyword" - }, - "host": { - "type": "keyword" - }, - "title": { - "type": "text" - }, - "url": { - "type": "keyword" - } - } - } -} diff --git a/external/opensearch-java/src/test/resources/metrics.mapping b/external/opensearch-java/src/test/resources/metrics.mapping deleted file mode 100644 index fc6ae3a09..000000000 --- a/external/opensearch-java/src/test/resources/metrics.mapping +++ /dev/null @@ -1,40 +0,0 @@ -{ - "index_patterns": "metrics*", - "settings": { - "index": { - "number_of_shards": 1, - "refresh_interval": "30s" - }, - "number_of_replicas": 0 - }, - "mappings": { - "_source": { "enabled": true }, - "properties": { - "name": { - "type": "keyword" - }, - "stormId": { - "type": "keyword" - }, - "srcComponentId": { - "type": "keyword" - }, - "srcTaskId": { - "type": "short" - }, - "srcWorkerHost": { - "type": "keyword" - }, - "srcWorkerPort": { - "type": "integer" - }, - "timestamp": { - "type": "date", - "format": "date_optional_time" - }, - "value": { - "type": "double" - } - } - } -} diff --git a/external/opensearch-java/src/test/resources/queues.mapping b/external/opensearch-java/src/test/resources/queues.mapping deleted file mode 100644 index b85f33dcc..000000000 --- a/external/opensearch-java/src/test/resources/queues.mapping +++ /dev/null @@ -1,33 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 10, - "number_of_replicas": 1, - "refresh_interval": "5s" - } - }, - "mappings": { - "dynamic_templates": [{ - "metadata": { - "path_match": "metadata.*", - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } - } - }], - "_source": { - "enabled": true - }, - "properties": { - "key": { - "type": "keyword", - "index": true - }, - "lastUpdated": { - "type": "date", - "format": "date_optional_time" - } - } - } -} diff --git a/external/opensearch-java/src/test/resources/status.mapping b/external/opensearch-java/src/test/resources/status.mapping deleted file mode 100644 index e5b14fe97..000000000 --- a/external/opensearch-java/src/test/resources/status.mapping +++ /dev/null @@ -1,39 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 10, - "number_of_replicas": 1, - "refresh_interval": "5s" - } - }, - "mappings": { - "dynamic_templates": [{ - "metadata": { - "path_match": "metadata.*", - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } - } - }], - "_source": { - "enabled": true - }, - "properties": { - "key": { - "type": "keyword", - "index": true - }, - "nextFetchDate": { - "type": "date", - "format": "date_optional_time" - }, - "status": { - "type": "keyword" - }, - "url": { - "type": "keyword" - } - } - } -} diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java similarity index 100% rename from external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java rename to external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java index e4eec09ef..c32f162f2 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java @@ -17,118 +17,55 @@ package org.apache.stormcrawler.opensearch; -import java.io.IOException; +import java.util.Objects; import org.jetbrains.annotations.NotNull; -import org.opensearch.action.DocWriteRequest; -import org.opensearch.action.DocWriteResponse; -import org.opensearch.action.bulk.BulkItemResponse; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentBuilder; +import org.jetbrains.annotations.Nullable; +import org.opensearch.client.opensearch._types.ErrorCause; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; -public final class BulkItemResponseToFailedFlag { - @NotNull public final BulkItemResponse response; - public final boolean failed; - @NotNull public final String id; +/** + * Wraps a {@link BulkResponseItem} with a pre-computed failure flag. A 409 (conflict) is not + * considered a failure — it simply indicates a document already existed when using create mode. + * + * @param response the original bulk response item + * @param failed whether this item represents a real failure (excludes 409 conflicts) + * @param id the document id from the response item + */ +public record BulkItemResponseToFailedFlag( + @NotNull BulkResponseItem response, boolean failed, @NotNull String id) { - public BulkItemResponseToFailedFlag(@NotNull BulkItemResponse response, boolean failed) { - this.response = response; - this.failed = failed; - this.id = response.getId(); + public BulkItemResponseToFailedFlag { + Objects.requireNonNull(response, "response"); + Objects.requireNonNull(id, "id"); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof BulkItemResponseToFailedFlag)) { - return false; - } - - BulkItemResponseToFailedFlag that = (BulkItemResponseToFailedFlag) o; - - if (failed != that.failed) { - return false; - } - if (!response.equals(that.response)) { - return false; - } - return id.equals(that.id); + /** Constructs with id derived from the response item. */ + public BulkItemResponseToFailedFlag(@NotNull BulkResponseItem response, boolean failed) { + this( + response, + failed, + Objects.requireNonNull(response.id(), "BulkResponseItem id must not be null")); } - @Override - public int hashCode() { - int result = response.hashCode(); - result = 31 * result + (failed ? 1 : 0); - result = 31 * result + id.hashCode(); - return result; + /** Returns the error cause, or {@code null} if the item did not fail. */ + @Nullable + public ErrorCause getFailedCause() { + return response.error(); } - @Override - public String toString() { - return "BulkItemResponseToFailedFlag{" - + "response=" - + response - + ", failed=" - + failed - + ", id='" - + id - + '\'' - + '}'; + /** Returns a human-readable failure description, or {@code null} if the item did not fail. */ + @Nullable + public String getFailure() { + ErrorCause error = response.error(); + if (error == null) { + return null; + } + return error.reason() != null ? error.reason() : error.type(); } - public RestStatus status() { + // opensearch-java: status() returns int HTTP code, not RestStatus enum + /** Returns the HTTP status code of this response item. */ + public int getStatus() { return response.status(); } - - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) - throws IOException { - return response.toXContent(builder, params); - } - - public int getItemId() { - return response.getItemId(); - } - - public DocWriteRequest.OpType getOpType() { - return response.getOpType(); - } - - public String getIndex() { - return response.getIndex(); - } - - public long getVersion() { - return response.getVersion(); - } - - public T getResponse() { - return response.getResponse(); - } - - public boolean isFailed() { - return response.isFailed(); - } - - public String getFailureMessage() { - return response.getFailureMessage(); - } - - public BulkItemResponse.Failure getFailure() { - return response.getFailure(); - } - - public void writeTo(StreamOutput out) throws IOException { - response.writeTo(out); - } - - public void writeThin(StreamOutput out) throws IOException { - response.writeThin(out); - } - - public boolean isFragment() { - return response.isFragment(); - } } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java index 4935bc1a3..fb03a9f0f 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java @@ -20,14 +20,14 @@ import com.fasterxml.jackson.databind.JsonNode; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.Timer; import java.util.TimerTask; import org.apache.stormcrawler.JSONResource; -import org.opensearch.action.get.GetRequest; -import org.opensearch.action.get.GetResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.GetResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +38,10 @@ * org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper} to eliminate duplicated * setup/refresh/cleanup logic. * + *

This is the opensearch-java (OpenSearch Java Client 3.x / HC5) counterpart of the class with + * the same name in the {@code external/opensearch} module. It uses the typed {@link + * OpenSearchClient} instead of the deprecated {@code RestHighLevelClient}. + * * @param the base type that the delegate must extend (e.g. URLFilter or ParseFilter) */ public class DelegateRefresher { @@ -46,7 +50,7 @@ public class DelegateRefresher { private final T delegate; private Timer refreshTimer; - private RestHighLevelClient osClient; + private OpenSearchClient osClient; /** * Creates a refresher by loading the delegate class from the JSON configuration. @@ -127,13 +131,18 @@ public void run() { if (osClient != null) { LOG.info("Reloading json resources from OpenSearch"); try { - GetResponse response = + GetResponse response = osClient.get( - new GetRequest( - "config", resource.getResourceFile()), - RequestOptions.DEFAULT); - resource.loadJSONResources( - new ByteArrayInputStream(response.getSourceAsBytes())); + g -> + g.index("config") + .id(resource.getResourceFile()), + JsonData.class); + if (response.found() && response.source() != null) { + String json = response.source().toJson().toString(); + resource.loadJSONResources( + new ByteArrayInputStream( + json.getBytes(StandardCharsets.UTF_8))); + } } catch (Exception e) { LOG.error("Can't load config from OpenSearch", e); } @@ -156,7 +165,7 @@ public void cleanup() { } if (osClient != null) { try { - osClient.close(); + osClient._transport().close(); } catch (IOException e) { LOG.error("Exception when closing OpenSearch client", e); } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java index 180a10743..f76172057 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java @@ -17,29 +17,22 @@ package org.apache.stormcrawler.opensearch; -import com.google.common.base.Charsets; import com.google.common.io.Resources; import java.io.IOException; import java.net.URL; -import org.opensearch.OpenSearchException; -import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.client.indices.CreateIndexRequest; -import org.opensearch.client.indices.CreateIndexResponse; -import org.opensearch.client.indices.GetIndexRequest; -import org.opensearch.client.indices.IndexTemplatesExistRequest; -import org.opensearch.client.indices.PutIndexTemplateRequest; -import org.opensearch.common.xcontent.XContentType; +import java.nio.charset.StandardCharsets; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.generic.Requests; +import org.opensearch.client.opensearch.generic.Response; +import org.opensearch.client.opensearch.indices.ExistsTemplateRequest; import org.slf4j.Logger; public class IndexCreation { public static synchronized void checkOrCreateIndex( - RestHighLevelClient client, String indexName, String boltType, Logger log) + OpenSearchClient client, String indexName, String boltType, Logger log) throws IOException { - final boolean indexExists = - client.indices().exists(new GetIndexRequest(indexName), RequestOptions.DEFAULT); + final boolean indexExists = client.indices().exists(req -> req.index(indexName)).value(); log.info("Index '{}' exists? {}", indexName, indexExists); // there's a possible check-then-update race condition // createIndex intentionally catches and logs exceptions from OpenSearch @@ -51,13 +44,12 @@ public static synchronized void checkOrCreateIndex( } public static synchronized void checkOrCreateIndexTemplate( - RestHighLevelClient client, String boltType, Logger log) throws IOException { + OpenSearchClient client, String boltType, Logger log) throws IOException { final String templateName = boltType + "-template"; final boolean templateExists = client.indices() - .existsTemplate( - new IndexTemplatesExistRequest(templateName), - RequestOptions.DEFAULT); + .existsTemplate(ExistsTemplateRequest.of(r -> r.name(templateName))) + .value(); log.info("Template '{}' exists? {}", templateName, templateExists); // there's a possible check-then-update race condition // createTemplate intentionally catches and logs exceptions from OpenSearch @@ -69,46 +61,54 @@ public static synchronized void checkOrCreateIndexTemplate( } private static boolean createTemplate( - RestHighLevelClient client, String templateName, String resourceName, Logger log) { + OpenSearchClient client, String templateName, String resourceName, Logger log) { try { - final PutIndexTemplateRequest createIndexRequest = - new PutIndexTemplateRequest(templateName); - final URL mapping = Thread.currentThread().getContextClassLoader().getResource(resourceName); - final String jsonIndexConfiguration = Resources.toString(mapping, Charsets.UTF_8); - - createIndexRequest.source(jsonIndexConfiguration, XContentType.JSON); - - final AcknowledgedResponse createIndexResponse = - client.indices().putTemplate(createIndexRequest, RequestOptions.DEFAULT); - return createIndexResponse.isAcknowledged(); - } catch (IOException | OpenSearchException e) { + final String jsonIndexConfiguration = + Resources.toString(mapping, StandardCharsets.UTF_8); + + try (Response response = + client.generic() + .execute( + Requests.builder() + .endpoint("/_template/" + templateName) + .method("PUT") + .json(jsonIndexConfiguration) + .build())) { + int statusCode = response.getStatus(); + return statusCode == 200 || statusCode == 201; + } + } catch (Exception e) { log.warn("template '{}' not created", templateName, e); return false; } } private static boolean createIndex( - RestHighLevelClient client, String indexName, String resourceName, Logger log) { + OpenSearchClient client, String indexName, String resourceName, Logger log) { try { - - final CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); - final URL mapping = Thread.currentThread().getContextClassLoader().getResource(resourceName); - final String jsonIndexConfiguration = Resources.toString(mapping, Charsets.UTF_8); - - createIndexRequest.source(jsonIndexConfiguration, XContentType.JSON); - - final CreateIndexResponse createIndexResponse = - client.indices().create(createIndexRequest, RequestOptions.DEFAULT); - return createIndexResponse.isAcknowledged(); - } catch (IOException | OpenSearchException e) { + final String jsonIndexConfiguration = + Resources.toString(mapping, StandardCharsets.UTF_8); + + try (Response response = + client.generic() + .execute( + Requests.builder() + .endpoint("/" + indexName) + .method("PUT") + .json(jsonIndexConfiguration) + .build())) { + int statusCode = response.getStatus(); + return statusCode == 200 || statusCode == 201; + } + } catch (Exception e) { log.warn("index '{}' not created", indexName, e); return false; } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java index f9a5dea1a..eeb473628 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java @@ -17,8 +17,8 @@ package org.apache.stormcrawler.opensearch; -import static org.opensearch.client.RestClientBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS; -import static org.opensearch.client.RestClientBuilder.DEFAULT_SOCKET_TIMEOUT_MILLIS; +import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS; +import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS; import java.io.IOException; import java.net.URI; @@ -30,32 +30,29 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; +import javax.net.ssl.SSLContext; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.Credentials; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.TrustAllStrategy; -import org.apache.http.ssl.SSLContextBuilder; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.reactor.ssl.TlsDetails; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.util.Timeout; import org.apache.stormcrawler.util.ConfUtils; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.opensearch.action.DocWriteRequest; -import org.opensearch.action.bulk.BulkProcessor; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.client.HttpAsyncResponseConsumerFactory; -import org.opensearch.client.Node; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestClient; -import org.opensearch.client.RestClientBuilder; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.client.sniff.OpenSearchNodesSniffer; -import org.opensearch.client.sniff.Sniffer; -import org.opensearch.common.unit.TimeValue; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchAsyncClient; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,24 +63,248 @@ public final class OpenSearchConnection { private static final Logger LOG = LoggerFactory.getLogger(OpenSearchConnection.class); - @NotNull private final RestHighLevelClient client; + @NotNull private final OpenSearchClient client; - @NotNull private final BulkProcessor processor; + @NotNull private final AsyncBulkProcessor processor; - @Nullable private final Sniffer sniffer; + @NotNull private final OpenSearchTransport transport; private OpenSearchConnection( - @NotNull RestHighLevelClient c, @NotNull BulkProcessor p, @Nullable Sniffer s) { - processor = p; + @NotNull OpenSearchClient c, + @NotNull AsyncBulkProcessor p, + @NotNull OpenSearchTransport t) { client = c; - sniffer = s; + processor = p; + transport = t; } - public RestHighLevelClient getClient() { + public OpenSearchClient getClient() { return client; } - public static RestHighLevelClient getClient(Map stormConf, String boltType) { + /** + * Creates a standalone {@link OpenSearchClient}. Used by classes that need a client without a + * bulk processor (e.g. spouts, filters). Callers are responsible for closing the returned + * client's transport via {@code client._transport().close()}. + */ + public static OpenSearchClient getClient(Map stormConf, String boltType) { + return buildClientResources(stormConf, boltType).client(); + } + + /** Adds a single bulk operation to the internal processor. */ + public void addToProcessor(final BulkOperation operation) { + processor.add(operation); + } + + /** + * Creates a connection with a default (no-op) listener. The values for bolt type are + * [indexer,status,metrics]. + */ + public static OpenSearchConnection getConnection( + Map stormConf, String boltType) { + AsyncBulkProcessor.Listener listener = + new AsyncBulkProcessor.Listener() { + @Override + public void afterBulk( + long arg0, + org.opensearch.client.opensearch.core.BulkRequest arg1, + org.opensearch.client.opensearch.core.BulkResponse arg2) {} + + @Override + public void afterBulk( + long arg0, + org.opensearch.client.opensearch.core.BulkRequest arg1, + Throwable arg2) {} + + @Override + public void beforeBulk( + long arg0, org.opensearch.client.opensearch.core.BulkRequest arg1) {} + }; + return getConnection(stormConf, boltType, listener); + } + + public static OpenSearchConnection getConnection( + Map stormConf, String boltType, AsyncBulkProcessor.Listener listener) { + + final String dottedType = boltType + "."; + + warnOnRemovedKeys(stormConf, dottedType); + + ClientResources cr = buildClientResources(stormConf, boltType); + + final String flushIntervalString = + ConfUtils.getString( + stormConf, Constants.PARAMPREFIX, dottedType, "flushInterval", "5s"); + + final long flushIntervalMillis = parseTimeValueToMillis(flushIntervalString, 5000); + + final int bulkActions = + ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "bulkActions", 50); + + final int concurrentRequests = + ConfUtils.getInt( + stormConf, Constants.PARAMPREFIX, dottedType, "concurrentRequests", 1); + + AsyncBulkProcessor bulkProcessor = null; + try { + bulkProcessor = + new AsyncBulkProcessor.Builder( + new OpenSearchAsyncClient(cr.transport()), listener) + .setBulkActions(bulkActions) + .setFlushIntervalMillis(flushIntervalMillis) + .setConcurrentRequests(concurrentRequests) + .build(); + + return new OpenSearchConnection(cr.client(), bulkProcessor, cr.transport()); + } catch (Exception e) { + if (bulkProcessor != null) { + try { + bulkProcessor.close(); + } catch (Exception suppressed) { + e.addSuppressed(suppressed); + } + } + try { + cr.transport().close(); + } catch (IOException suppressed) { + e.addSuppressed(suppressed); + } + throw e; + } + } + + private final AtomicBoolean isClosed = new AtomicBoolean(false); + + public void close() { + + if (!isClosed.compareAndSet(false, true)) { + LOG.warn("Tried to close an already closed connection!"); + return; + } + + LOG.debug("Start closing the OpenSearch connection"); + + // First, close the BulkProcessor ensuring pending actions are flushed + try { + boolean success = processor.awaitClose(60, TimeUnit.SECONDS); + if (!success) { + throw new RuntimeException( + "Failed to flush pending actions when closing BulkProcessor"); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + // Now close the transport (also shuts down the underlying HTTP client) + try { + transport.close(); + } catch (IOException e) { + LOG.trace("Transport threw IO exception on close."); + } + } + + /** + * Extracts the document ID from a {@link BulkOperation} regardless of its type (index, create, + * delete, update). + */ + public static String getBulkOperationId(BulkOperation op) { + if (op.isIndex()) { + return op.index().id(); + } + if (op.isCreate()) { + return op.create().id(); + } + if (op.isDelete()) { + return op.delete().id(); + } + if (op.isUpdate()) { + return op.update().id(); + } + return null; + } + + /** + * Returns the scopes the Basic credentials are registered for: the scheme, host and port of + * each configured address. A request to any other address does not receive them. + */ + static List credentialScopes(List hosts) { + final Set scopes = new LinkedHashSet<>(); + for (HttpHost host : hosts) { + scopes.add(new AuthScope(host)); + } + return new ArrayList<>(scopes); + } + + /** + * Returns the addresses which use plain http and are not a loopback address, i.e. those the + * Basic credentials would be sent to in the clear over the network. + */ + static List plainHttpHosts(List hosts) { + final List plain = new ArrayList<>(); + for (HttpHost host : hosts) { + if ("http".equalsIgnoreCase(host.getSchemeName()) && !isLoopback(host.getHostName())) { + plain.add(host); + } + } + return plain; + } + + private static final Pattern IPV4_LOOPBACK = Pattern.compile("127(\\.\\d{1,3}){3}"); + + /** Whether the host is localhost or a loopback IP literal. No name is resolved. */ + static boolean isLoopback(String hostname) { + String host = StringUtils.strip(hostname.toLowerCase(Locale.ROOT), "[]"); + return host.equals("localhost") + || IPV4_LOOPBACK.matcher(host).matches() + || host.equals("::1") + || host.equals("0:0:0:0:0:0:0:1"); + } + + private static void warnAboutPlainHttp(String boltType, List hosts) { + final List plain = plainHttpHosts(hosts); + if (!plain.isEmpty()) { + LOG.warn( + "OpenSearch credentials are configured for {} but the addresses {} use plain " + + "http, the credentials are sent unencrypted. Give the addresses an " + + "https:// scheme.", + boltType, + plain); + } + } + + // internal helpers + private record ClientResources(OpenSearchClient client, OpenSearchTransport transport) {} + + /** + * Logs a WARN for legacy configuration keys that are no longer honoured by this module, so that + * users migrating from {@code external/opensearch} notice silently-dropped tuning. See the + * module README for the full list of differences. + */ + private static void warnOnRemovedKeys(Map stormConf, String dottedType) { + final String responseBufferKey = Constants.PARAMPREFIX + dottedType + "responseBufferSize"; + if (stormConf.containsKey(responseBufferKey)) { + LOG.warn( + "Configuration key '{}' is set but no longer supported by the opensearch-java module. " + + "The HC5-based async transport does not expose an equivalent per-request " + + "heap-buffer override. The setting is ignored — remove it from your " + + "configuration. See external/opensearch-java/README.md for details.", + responseBufferKey); + } + + final String sniffKey = Constants.PARAMPREFIX + dottedType + "sniff"; + if (stormConf.containsKey(sniffKey)) { + LOG.warn( + "Configuration key '{}' is set but no longer supported by the opensearch-java module. " + + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so " + + "automatic node discovery is not available. Keep the 'addresses' list up to " + + "date manually or put a load balancer in front of the cluster. " + + "See external/opensearch-java/README.md for details.", + sniffKey); + } + } + + private static ClientResources buildClientResources( + Map stormConf, String boltType) { final String dottedType = boltType + "."; @@ -119,10 +340,15 @@ public static RestHighLevelClient getClient(Map stormConf, Strin if (uri.getScheme() != null) { scheme = uri.getScheme(); } - hosts.add(new HttpHost(uri.getHost(), port, scheme)); + // HC5: constructor is (scheme, hostname, port) — not (hostname, port, scheme) + hosts.add(new HttpHost(scheme, uri.getHost(), port)); } - final RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[0])); + LOG.info( + "OpenSearch {} transport configured with {} host(s): {}", + boltType, + hosts.size(), + hosts); // authentication via user / password final String user = @@ -144,9 +370,40 @@ public static RestHighLevelClient getClient(Map stormConf, Strin ConfUtils.getBoolean( stormConf, Constants.PARAMPREFIX, "", "disable.tls.validation", false); - final boolean needsUser = hasCredentials(user, password); + final boolean needsUser = StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password); final boolean needsProxy = StringUtils.isNotBlank(proxyhost) && proxyport != -1; + // Defaults from ApacheHttpClient5TransportBuilder (same as the former RestClientBuilder) + final int connectTimeout = + ConfUtils.getInt( + stormConf, + Constants.PARAMPREFIX, + dottedType, + "connect.timeout", + DEFAULT_CONNECT_TIMEOUT_MILLIS); + final int socketTimeout = + ConfUtils.getInt( + stormConf, + Constants.PARAMPREFIX, + dottedType, + "socket.timeout", + DEFAULT_RESPONSE_TIMEOUT_MILLIS); + + final boolean compression = + ConfUtils.getBoolean( + stormConf, Constants.PARAMPREFIX, dottedType, "compression", false); + + final ApacheHttpClient5TransportBuilder builder = + ApacheHttpClient5TransportBuilder.builder(hosts.toArray(new HttpHost[0])) + .setMapper(new JacksonJsonpMapper()); + + // Timeouts via ConnectionConfig on the builder's internal connection manager + builder.setConnectionConfigCallback( + connConfigBuilder -> + connConfigBuilder + .setConnectTimeout(Timeout.ofMilliseconds(connectTimeout)) + .setSocketTimeout(Timeout.ofMilliseconds(socketTimeout))); + if (needsUser) { warnAboutPlainHttp(boltType, hosts); } @@ -157,27 +414,61 @@ public static RestHighLevelClient getClient(Map stormConf, Strin boltType); } + // Auth, proxy, and/or trust-all SSL via HttpClient customisation if (needsUser || needsProxy || disableTlsValidation) { builder.setHttpClientConfigCallback( httpClientBuilder -> { + // hc.client5 auth: password is char[], AuthScope(host, port) if (needsUser) { - httpClientBuilder.setDefaultCredentialsProvider( - new OriginCredentialsProvider( - hosts, - new UsernamePasswordCredentials(user, password))); + final BasicCredentialsProvider credentialsProvider = + new BasicCredentialsProvider(); + final UsernamePasswordCredentials credentials = + new UsernamePasswordCredentials(user, password.toCharArray()); + for (AuthScope scope : credentialScopes(hosts)) { + credentialsProvider.setCredentials(scope, credentials); + } + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); } + // hc.client5 proxy: HttpHost(scheme, host, port) if (needsProxy) { httpClientBuilder.setProxy( - new HttpHost(proxyhost, proxyport, proxyscheme)); + new HttpHost(proxyscheme, proxyhost, proxyport)); } - + // Custom connection manager overrides the builder's internal one, + // so timeouts and TlsDetailsFactory must be replicated here if (disableTlsValidation) { try { - final SSLContextBuilder sslContext = new SSLContextBuilder(); - sslContext.loadTrustMaterial(null, new TrustAllStrategy()); - httpClientBuilder.setSSLContext(sslContext.build()); - httpClientBuilder.setSSLHostnameVerifier( - NoopHostnameVerifier.INSTANCE); + final SSLContext sslContext = + SSLContextBuilder.create() + .loadTrustMaterial((chain, authType) -> true) + .build(); + httpClientBuilder.setConnectionManager( + PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy( + ClientTlsStrategyBuilder.create() + .setSslContext(sslContext) + .setHostnameVerifier( + NoopHostnameVerifier + .INSTANCE) + // HTTP/2 ALPN negotiation + .setTlsDetailsFactory( + sslEngine -> + new TlsDetails( + sslEngine + .getSession(), + sslEngine + .getApplicationProtocol())) + .build()) + .setDefaultConnectionConfig( + ConnectionConfig.custom() + .setConnectTimeout( + Timeout.ofMilliseconds( + connectTimeout)) + .setSocketTimeout( + Timeout.ofMilliseconds( + socketTimeout)) + .build()) + .build()); } catch (Exception e) { throw new RuntimeException("Failed to disable TLS validation", e); } @@ -186,324 +477,40 @@ public static RestHighLevelClient getClient(Map stormConf, Strin }); } - final int connectTimeout = - ConfUtils.getInt( - stormConf, - Constants.PARAMPREFIX, - dottedType, - "connect.timeout", - DEFAULT_CONNECT_TIMEOUT_MILLIS); - final int socketTimeout = - ConfUtils.getInt( - stormConf, - Constants.PARAMPREFIX, - dottedType, - "socket.timeout", - DEFAULT_SOCKET_TIMEOUT_MILLIS); - // timeout until connection is established - builder.setRequestConfigCallback( - requestConfigBuilder -> - requestConfigBuilder - .setConnectTimeout(connectTimeout) - // Timeout when waiting for data - .setSocketTimeout(socketTimeout)); - - // TODO check if this has gone somewhere else - // int maxRetryTimeout = ConfUtils.getInt(stormConf, Constants.PARAMPREFIX + - // boltType + - // ".max.retry.timeout", - // DEFAULT_MAX_RETRY_TIMEOUT_MILLIS); - // builder.setMaxRetryTimeoutMillis(maxRetryTimeout); - - // TODO configure headers etc... - // Map configSettings = (Map) stormConf - // .get(Constants.PARAMPREFIX + boltType + ".settings"); - // if (configSettings != null) { - // configSettings.forEach((k, v) -> settings.put(k, v)); - // } - - // use node selector only to log nodes listed in the config - // and/or discovered through sniffing - builder.setNodeSelector( - nodes -> { - for (Node node : nodes) { - LOG.debug( - "Connected to OpenSearch node {} [{}] for {}", - node.getName(), - node.getHost(), - boltType); - } - }); - - final boolean compression = - ConfUtils.getBoolean( - stormConf, Constants.PARAMPREFIX, dottedType, "compression", false); - + // Compression: first-class builder method, not a request interceptor builder.setCompressionEnabled(compression); - return new RestHighLevelClient(builder); - } + final OpenSearchTransport transport = builder.build(); + final OpenSearchClient openSearchClient = new OpenSearchClient(transport); - /** Basic authentication is only set up when both the user and the password are given. */ - static boolean hasCredentials(String user, String password) { - return StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password); + return new ClientResources(openSearchClient, transport); } /** - * Gives the Basic credentials only to requests for one of the configured addresses, matched on - * scheme, host and port. A node reached under any other address, such as one found by the - * sniffer under the address it publishes, does not receive them. + * Parses a time value string (e.g. "5s", "500ms", "1m") into milliseconds. + * + * @param value the string to parse + * @param defaultMillis the default if parsing fails + * @return milliseconds */ - static final class OriginCredentialsProvider implements CredentialsProvider { - - private final Set origins = new LinkedHashSet<>(); - private final Credentials credentials; - - OriginCredentialsProvider(List hosts, Credentials credentials) { - for (HttpHost host : hosts) { - origins.add(origin(host)); - } - this.credentials = credentials; - } - - private static String origin(HttpHost host) { - final String scheme = host.getSchemeName().toLowerCase(Locale.ROOT); - int port = host.getPort(); - if (port < 0) { - port = "https".equals(scheme) ? 443 : 80; - } - return scheme + "://" + host.getHostName().toLowerCase(Locale.ROOT) + ":" + port; - } - - @Override - public Credentials getCredentials(AuthScope scope) { - final HttpHost origin = scope.getOrigin(); - if (origin != null && origins.contains(origin(origin))) { - return credentials; - } - return null; + static long parseTimeValueToMillis(String value, long defaultMillis) { + if (value == null || value.isBlank()) { + return defaultMillis; } - - @Override - public void setCredentials(AuthScope scope, Credentials credentials) { - throw new UnsupportedOperationException(); - } - - @Override - public void clear() {} - } - - /** - * Returns the addresses which use plain http and are not a loopback address, i.e. those the - * Basic credentials would be sent to in the clear over the network. - */ - static List plainHttpHosts(List hosts) { - final List plain = new ArrayList<>(); - for (HttpHost host : hosts) { - if ("http".equalsIgnoreCase(host.getSchemeName()) && !isLoopback(host.getHostName())) { - plain.add(host); - } - } - return plain; - } - - /** - * Returns the scheme under which the sniffer registers the nodes it finds. The nodes publish no - * scheme, so https is used as soon as one configured address uses it; otherwise sniffing would - * move the requests to plain http after the first round. - */ - static OpenSearchNodesSniffer.Scheme sniffScheme(List hosts) { - for (HttpHost host : hosts) { - if ("https".equalsIgnoreCase(host.getSchemeName())) { - return OpenSearchNodesSniffer.Scheme.HTTPS; - } - } - return OpenSearchNodesSniffer.Scheme.HTTP; - } - - private static final Pattern IPV4_LOOPBACK = Pattern.compile("127(\\.\\d{1,3}){3}"); - - /** Whether the host is localhost or a loopback IP literal. No name is resolved. */ - static boolean isLoopback(String hostname) { - String host = StringUtils.strip(hostname.toLowerCase(Locale.ROOT), "[]"); - return host.equals("localhost") - || IPV4_LOOPBACK.matcher(host).matches() - || host.equals("::1") - || host.equals("0:0:0:0:0:0:0:1"); - } - - private static void warnAboutPlainHttp(String boltType, List hosts) { - final List plain = plainHttpHosts(hosts); - if (!plain.isEmpty()) { - LOG.warn( - "OpenSearch credentials are configured for {} but the addresses {} use plain " - + "http, the credentials are sent unencrypted. Give the addresses an " - + "https:// scheme.", - boltType, - plain); - } - } - - public void addToProcessor(final DocWriteRequest request) { - processor.add(request); - } - - /** - * Creates a connection with a default listener. The values for bolt type are - * [indexer,status,metrics] - */ - public static OpenSearchConnection getConnection( - Map stormConf, String boltType) { - BulkProcessor.Listener listener = - new BulkProcessor.Listener() { - @Override - public void afterBulk(long arg0, BulkRequest arg1, BulkResponse arg2) {} - - @Override - public void afterBulk(long arg0, BulkRequest arg1, Throwable arg2) {} - - @Override - public void beforeBulk(long arg0, BulkRequest arg1) {} - }; - return getConnection(stormConf, boltType, listener); - } - - public static OpenSearchConnection getConnection( - Map stormConf, String boltType, BulkProcessor.Listener listener) { - - final RestHighLevelClient client = getClient(stormConf, boltType); - - final String dottedType = boltType + "."; - - final String flushIntervalString = - ConfUtils.getString( - stormConf, Constants.PARAMPREFIX, dottedType, "flushInterval", "5s"); - - final TimeValue flushInterval = - TimeValue.parseTimeValue( - flushIntervalString, TimeValue.timeValueSeconds(5), "flushInterval"); - - final int bulkActions = - ConfUtils.getInt(stormConf, Constants.PARAMPREFIX, dottedType, "bulkActions", 50); - - final int concurrentRequests = - ConfUtils.getInt( - stormConf, Constants.PARAMPREFIX, dottedType, "concurrentRequests", 1); - - final RequestOptions requestOptions = RequestOptions.DEFAULT; - final RequestOptions.Builder requestOptionsBuilder = requestOptions.toBuilder(); - final int bufferSize = - ConfUtils.getInt( - stormConf, Constants.PARAMPREFIX, dottedType, "responseBufferSize", 100); - - requestOptionsBuilder.setHttpAsyncResponseConsumerFactory( - new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory( - bufferSize * 1024 * 1024)); - - BulkProcessor bulkProcessor = null; - Sniffer sniffer = null; - try { - bulkProcessor = - BulkProcessor.builder( - (request, bulkListener) -> - client.bulkAsync( - request, - requestOptionsBuilder.build(), - bulkListener), - listener) - .setFlushInterval(flushInterval) - .setBulkActions(bulkActions) - .setConcurrentRequests(concurrentRequests) - .build(); - - boolean sniff = - ConfUtils.getBoolean( - stormConf, Constants.PARAMPREFIX, dottedType, "sniff", true); - if (sniff) { - if (hasCredentials( - ConfUtils.getString(stormConf, Constants.PARAMPREFIX, dottedType, "user"), - ConfUtils.getString( - stormConf, Constants.PARAMPREFIX, dottedType, "password"))) { - LOG.warn( - "Sniffing is enabled for {} and OpenSearch credentials are configured. " - + "The credentials are only sent to the configured addresses: " - + "requests to a node the sniffer finds under another host or " - + "port are sent without them and fail if the cluster requires " - + "authentication. List every node in opensearch.{}.addresses " - + "or set opensearch.{}.sniff: false.", - boltType, - boltType, - boltType); - } - final RestClient lowLevelClient = client.getLowLevelClient(); - final List configured = new ArrayList<>(); - for (Node node : lowLevelClient.getNodes()) { - configured.add(node.getHost()); - } - sniffer = - Sniffer.builder(lowLevelClient) - .setNodesSniffer( - new OpenSearchNodesSniffer( - lowLevelClient, - OpenSearchNodesSniffer - .DEFAULT_SNIFF_REQUEST_TIMEOUT, - sniffScheme(configured))) - .build(); - } - - return new OpenSearchConnection(client, bulkProcessor, sniffer); - } catch (Exception e) { - if (bulkProcessor != null) { - try { - bulkProcessor.close(); - } catch (Exception suppressed) { - e.addSuppressed(suppressed); - } - } - try { - client.close(); - } catch (IOException suppressed) { - e.addSuppressed(suppressed); - } - throw e; - } - } - - private boolean isClosed = false; - - public void close() { - - if (isClosed) { - LOG.warn("Tried to close an already closed connection!"); - return; - } - - // Maybe some kind of identifier? - LOG.debug("Start closing the OpenSearch connection"); - - // First, close the BulkProcessor ensuring pending actions are flushed + value = value.trim(); try { - boolean success = processor.awaitClose(60, TimeUnit.SECONDS); - if (!success) { - throw new RuntimeException( - "Failed to flush pending actions when closing BulkProcessor"); + if (value.endsWith("ms")) { + return Long.parseLong(value.substring(0, value.length() - 2)); + } else if (value.endsWith("s")) { + return Long.parseLong(value.substring(0, value.length() - 1)) * 1000; + } else if (value.endsWith("m")) { + return Long.parseLong(value.substring(0, value.length() - 1)) * 60000; + } else { + return Long.parseLong(value); } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - if (sniffer != null) { - sniffer.close(); + } catch (NumberFormatException e) { + LOG.warn("Could not parse time value '{}', using default {}ms", value, defaultMillis); + return defaultMillis; } - - // Now close the actual client - try { - client.close(); - } catch (IOException e) { - // ignore silently - LOG.trace("Client threw IO exception."); - } - - isClosed = true; } } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java index bdb9a8dca..0e8574fbb 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java @@ -21,23 +21,22 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.RemovalCause; import com.github.benmanes.caffeine.cache.Ticker; -import java.util.Arrays; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.metrics.ScopedCounter; import org.jetbrains.annotations.Nullable; -import org.opensearch.action.DocWriteRequest; -import org.opensearch.action.bulk.BulkItemResponse; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.core.rest.RestStatus; +// opensearch-java: uses typed BulkRequest/BulkResponse, not legacy REST equivalents +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; import org.slf4j.Logger; /** @@ -54,8 +53,7 @@ public interface TupleAction { } private final Cache> cache; - private final java.util.concurrent.locks.ReentrantLock lock = - new java.util.concurrent.locks.ReentrantLock(true); + private final ReentrantLock lock = new ReentrantLock(true); private final Logger log; private final Consumer onEviction; @@ -64,7 +62,10 @@ public WaitAckCache(Logger log, Consumer onEviction) { this(Caffeine.newBuilder().expireAfterWrite(60, TimeUnit.SECONDS), log, onEviction); } - /** Creates a cache from a Caffeine spec string (e.g. "expireAfterWrite=300s"). */ + /** + * Creates a cache using a {@link Caffeine} spec string (e.g. {@code "expireAfterWrite=300s"}), + * typically driven by {@code topology.message.timeout.secs}. + */ public WaitAckCache(String cacheSpec, Logger log, Consumer onEviction) { this(Caffeine.from(cacheSpec), log, onEviction); } @@ -98,6 +99,7 @@ private WaitAckCache(Caffeine builder, Logger log, Consumer; status() returns int var idsToBulkItems = - Arrays.stream(response.getItems()) + response.items().stream() .map( bir -> { - BulkItemResponse.Failure f = bir.getFailure(); + var error = bir.error(); boolean failed = false; - if (f != null) { - if (f.getStatus().equals(RestStatus.CONFLICT)) { + if (error != null) { + // opensearch-java: int status code, not RestStatus enum + if (bir.status() == 409) { if (conflictCounter != null) { conflictCounter.scope("doc_conflicts").incrBy(1); } - log.debug("Doc conflict ID {}", bir.getId()); + log.debug("Doc conflict ID {}", bir.id()); } else { log.error( - "Bulk item failure ID {}: {}", bir.getId(), f); + "Bulk item failure ID {}: {}", + bir.id(), + error.reason() != null + ? error.reason() + : error.type()); failed = true; } } @@ -198,7 +206,9 @@ public void processBulkResponse( }) .collect( // https://github.com/apache/stormcrawler/issues/832 - Collectors.groupingBy(b -> b.id, Collectors.toUnmodifiableList())); + Collectors.groupingBy( + BulkItemResponseToFailedFlag::id, + Collectors.toUnmodifiableList())); Map> presentTuples; long estimatedSize; @@ -230,7 +240,7 @@ public void processBulkResponse( if (tuples != null) { log.debug("Found {} tuple(s) for ID {}", tuples.size(), id); for (Tuple t : tuples) { - if (selected.failed) { + if (selected.failed()) { failureCount++; } else { ackCount++; @@ -257,15 +267,22 @@ public void processBulkResponse( } } - /** Processes a failed bulk request by failing all associated tuples. */ + /** + * Processes a failed bulk request by failing all associated tuples. + * + * @param failAction callback applied to each tuple that must be failed + */ public void processFailedBulk( BulkRequest request, long executionId, Throwable failure, Consumer failAction) { log.error("Exception with bulk {} - failing the whole lot ", executionId, failure); + // opensearch-java: operations() + getBulkOperationId replaces + // legacy requests() + DocWriteRequest::id final var failedIds = - request.requests().stream() - .map(DocWriteRequest::id) + request.operations().stream() + .map(OpenSearchConnection::getBulkOperationId) + .filter(Objects::nonNull) .collect(Collectors.toUnmodifiableSet()); Map> failedTupleLists; @@ -306,7 +323,7 @@ private BulkItemResponseToFailedFlag selectBest( BulkItemResponseToFailedFlag best = items.get(0); int failedCount = 0; for (var item : items) { - if (item.failed) { + if (item.failed()) { failedCount++; } else { best = item; diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java index 16b3a0059..f7f0f1730 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java @@ -28,13 +28,13 @@ import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.indexing.AbstractIndexerBolt; import org.apache.stormcrawler.metrics.CrawlerMetrics; +import org.apache.stormcrawler.opensearch.AsyncBulkProcessor; import org.apache.stormcrawler.opensearch.OpenSearchConnection; import org.apache.stormcrawler.opensearch.WaitAckCache; import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.action.bulk.BulkProcessor.Listener; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; import org.slf4j.LoggerFactory; /** @@ -43,7 +43,7 @@ * will also try to delete documents even though they were never indexed and it currently won't * delete documents which were indexed under the canonical URL. */ -public class DeletionBolt extends BaseRichBolt implements Listener { +public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener { static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -81,7 +81,7 @@ public void prepare( try { connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this); } catch (Exception e1) { - LOG.error("Can't connect to opensearch", e1); + LOG.error("Can't connect to OpenSearch", e1); throw new RuntimeException(e1); } @@ -106,10 +106,18 @@ public void execute(Tuple tuple) { // used final String docID = getDocumentID(metadata, url); - DeleteRequest dr = new DeleteRequest(getIndexName(metadata), docID); - connection.addToProcessor(dr); + final String targetIndex = getIndexName(metadata); + BulkOperation op = BulkOperation.of(b -> b.delete(d -> d.index(targetIndex).id(docID))); waitAck.addTuple(docID, tuple); + + try { + connection.addToProcessor(op); + } catch (Exception e) { + LOG.error("Error adding the deletion of {} to the bulk", url, e); + waitAck.invalidate(docID); + _collector.fail(tuple); + } } @Override @@ -155,9 +163,9 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon waitAck.processBulkResponse( response, executionId, - null, // no conflict counter — deletion conflicts are expected and only logged + null, (id, t, selected) -> { - if (!selected.failed) { + if (!selected.failed()) { _collector.ack(t); } else { String url = (String) t.getValueByField("url"); diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java index 6191f2c3a..f7a441855 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java @@ -18,9 +18,10 @@ package org.apache.stormcrawler.opensearch.bolt; import static org.apache.stormcrawler.Constants.StatusStreamName; -import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.lang3.StringUtils; @@ -34,17 +35,15 @@ import org.apache.stormcrawler.metrics.CrawlerMetrics; import org.apache.stormcrawler.metrics.ScopedCounter; import org.apache.stormcrawler.metrics.ScopedReducedMetric; +import org.apache.stormcrawler.opensearch.AsyncBulkProcessor; import org.apache.stormcrawler.opensearch.IndexCreation; import org.apache.stormcrawler.opensearch.OpenSearchConnection; import org.apache.stormcrawler.opensearch.WaitAckCache; import org.apache.stormcrawler.persistence.Status; import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.action.bulk.BulkProcessor; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,7 +51,7 @@ * Sends documents to opensearch. Indexes all the fields from the tuples or a Map * <String,Object> from a named field. */ -public class IndexerBolt extends AbstractIndexerBolt implements BulkProcessor.Listener { +public class IndexerBolt extends AbstractIndexerBolt implements AsyncBulkProcessor.Listener { private static final Logger LOG = LoggerFactory.getLogger(IndexerBolt.class); @@ -105,7 +104,7 @@ public void prepare( try { connection = OpenSearchConnection.getConnection(conf, OSBoltType, this); } catch (Exception e1) { - LOG.error("Can't connect to opensearch", e1); + LOG.error("Can't connect to OpenSearch", e1); throw new RuntimeException(e1); } @@ -159,19 +158,19 @@ public void execute(Tuple tuple) { final String docID = getDocumentID(metadata, normalisedurl); try { - final XContentBuilder builder = jsonBuilder().startObject(); + final Map source = new HashMap<>(); // display text of the document? if (StringUtils.isNotBlank(fieldNameForText())) { final String text = trimText(tuple.getStringByField("text")); if (!ignoreEmptyFields() || StringUtils.isNotBlank(text)) { - builder.field(fieldNameForText(), trimText(text)); + source.put(fieldNameForText(), trimText(text)); } } // send URL as field? if (StringUtils.isNotBlank(fieldNameForURL())) { - builder.field(fieldNameForURL(), normalisedurl); + source.put(fieldNameForURL(), normalisedurl); } // which metadata to display? @@ -181,35 +180,54 @@ public void execute(Tuple tuple) { if (entry.getValue().length == 1) { final String value = entry.getValue()[0]; if (!ignoreEmptyFields() || StringUtils.isNotBlank(value)) { - builder.field(entry.getKey(), value); + source.put(entry.getKey(), value); } } else if (entry.getValue().length > 1) { - builder.array(entry.getKey(), entry.getValue()); + source.put(entry.getKey(), Arrays.asList(entry.getValue())); } } - builder.endObject(); - - final IndexRequest indexRequest = - new IndexRequest(getIndexName(metadata)) - .source(builder) - .id(docID) - .create(create); - - if (pipeline != null) { - indexRequest.setPipeline(pipeline); + final String targetIndex = getIndexName(metadata); + final BulkOperation op; + if (create) { + op = + BulkOperation.of( + b -> + b.create( + c -> { + c.index(targetIndex).id(docID).document(source); + if (pipeline != null) { + c.pipeline(pipeline); + } + return c; + })); + } else { + op = + BulkOperation.of( + b -> + b.index( + idx -> { + idx.index(targetIndex) + .id(docID) + .document(source); + if (pipeline != null) { + idx.pipeline(pipeline); + } + return idx; + })); } - connection.addToProcessor(indexRequest); + waitAck.addTuple(docID, tuple); + + connection.addToProcessor(op); eventCounter.scope("Indexed").incrBy(1); perSecMetrics.scope("Indexed").update(1); - - waitAck.addTuple(docID, tuple); - } catch (IOException e) { + } catch (Exception e) { LOG.error("Error building document for OpenSearch", e); // do not send to status stream so that it gets replayed _collector.fail(tuple); + waitAck.invalidate(docID); } } @@ -230,7 +248,7 @@ public void beforeBulk(long executionId, BulkRequest request) { @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { eventCounter.scope("bulks_received").incrBy(1); - eventCounter.scope("bulk_msec").incrBy(response.getTook().getMillis()); + eventCounter.scope("bulk_msec").incrBy(response.took()); waitAck.processBulkResponse( response, @@ -240,7 +258,7 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon String url = (String) t.getValueByField("url"); Metadata metadata = (Metadata) t.getValueByField("metadata"); - if (!selected.failed) { + if (!selected.failed()) { _collector.emit( StatusStreamName, t, new Values(url, metadata, Status.FETCHED)); _collector.ack(t); @@ -249,14 +267,16 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon LOG.error("update ID {}, URL {}, failure: {}", id, url, failure); // there is something wrong with the content we should // treat it as an ERROR - if (failure.getStatus().equals(RestStatus.BAD_REQUEST)) { + if (selected.getStatus() == 400) { metadata.setValue(Constants.STATUS_ERROR_SOURCE, "OpenSearch indexing"); metadata.setValue(Constants.STATUS_ERROR_MESSAGE, "invalid content"); _collector.emit( StatusStreamName, t, new Values(url, metadata, Status.ERROR)); _collector.ack(t); + LOG.debug("Acked {} with ID {}", url, id); } else { _collector.fail(t); + LOG.debug("Failed {} with ID {}", url, id); } } }); @@ -265,7 +285,6 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon @Override public void afterBulk(long executionId, BulkRequest request, Throwable failure) { eventCounter.scope("bulks_received").incrBy(1); - waitAck.processFailedBulk( request, executionId, diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java index 345025fb4..49ee7f0ca 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/filtering/JSONURLFilterWrapper.java @@ -63,10 +63,7 @@ public class JSONURLFilterWrapper extends URLFilter { public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) { refresher = new DelegateRefresher<>( - URLFilter.class, - stormConf, - filterParams, - (delegate, conf, params) -> delegate.configure(conf, params)); + URLFilter.class, stormConf, filterParams, URLFilter::configure); } @Override @@ -79,6 +76,8 @@ public void configure(@NotNull Map stormConf, @NotNull JsonNode @Override public void cleanup() { - refresher.cleanup(); + if (refresher != null) { + refresher.cleanup(); + } } } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java index 6b9ccf4cb..c46a2b734 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java @@ -17,12 +17,11 @@ package org.apache.stormcrawler.opensearch.metrics; -import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; - import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Date; +import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; @@ -32,8 +31,7 @@ import org.apache.stormcrawler.opensearch.IndexCreation; import org.apache.stormcrawler.opensearch.OpenSearchConnection; import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -144,19 +142,20 @@ private String getIndexName(Date timestamp) { private void indexDataPoint(TaskInfo taskInfo, Date timestamp, String name, double value) { try { - XContentBuilder builder = jsonBuilder().startObject(); - builder.field("stormId", stormID); - builder.field("srcComponentId", taskInfo.srcComponentId); - builder.field("srcTaskId", taskInfo.srcTaskId); - builder.field("srcWorkerHost", taskInfo.srcWorkerHost); - builder.field("srcWorkerPort", taskInfo.srcWorkerPort); - builder.field("name", name); - builder.field("value", value); - builder.field("timestamp", timestamp); - builder.endObject(); - - IndexRequest indexRequest = new IndexRequest(getIndexName(timestamp)).source(builder); - connection.addToProcessor(indexRequest); + Map doc = new HashMap<>(); + doc.put("stormId", stormID); + doc.put("srcComponentId", taskInfo.srcComponentId); + doc.put("srcTaskId", taskInfo.srcTaskId); + doc.put("srcWorkerHost", taskInfo.srcWorkerHost); + doc.put("srcWorkerPort", taskInfo.srcWorkerPort); + doc.put("name", name); + doc.put("value", value); + doc.put("timestamp", timestamp.toInstant().toString()); + + final String targetIndex = getIndexName(timestamp); + BulkOperation op = + BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc))); + connection.addToProcessor(op); } catch (Exception e) { LOG.error("problem when building request for OpenSearch", e); } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java index 3957ba6ab..d3b654a03 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsReporter.java @@ -17,8 +17,6 @@ package org.apache.stormcrawler.opensearch.metrics; -import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; - import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; @@ -30,6 +28,7 @@ import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.SortedMap; @@ -38,8 +37,7 @@ import org.apache.stormcrawler.opensearch.IndexCreation; import org.apache.stormcrawler.opensearch.OpenSearchConnection; import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,6 +66,8 @@ public class MetricsReporter extends ScheduledStormReporter { private static final String DATE_FORMAT_KEY = "opensearch.metrics.date.format"; + private ScheduledReporter reporter; + @Override public void prepare( MetricRegistry metricsRegistry, @@ -107,8 +107,6 @@ public void prepare( reporter.start(reportPeriod, reportPeriodUnit); } - private ScheduledReporter reporter; - @Override public void start() { // already started in prepare() @@ -123,7 +121,7 @@ public void stop() { /** * Inner ScheduledReporter that writes Codahale metrics to OpenSearch in the same format as the - * V1 MetricsConsumer. + * V1 {@link MetricsConsumer}. */ private static class OpenSearchScheduledReporter extends ScheduledReporter { @@ -203,16 +201,16 @@ private String getIndexName(Date timestamp) { private void indexDataPoint(Date timestamp, String name, double value) { try { - XContentBuilder builder = jsonBuilder().startObject(); - builder.field("stormId", stormId); - builder.field("name", name); - builder.field("value", value); - builder.field("timestamp", timestamp); - builder.endObject(); - - IndexRequest indexRequest = - new IndexRequest(getIndexName(timestamp)).source(builder); - connection.addToProcessor(indexRequest); + Map doc = new HashMap<>(); + doc.put("stormId", stormId); + doc.put("name", name); + doc.put("value", value); + doc.put("timestamp", timestamp.toInstant().toString()); + + final String targetIndex = getIndexName(timestamp); + BulkOperation op = + BulkOperation.of(b -> b.index(idx -> idx.index(targetIndex).document(doc))); + connection.addToProcessor(op); } catch (Exception e) { LOG.error("Problem when building request for OpenSearch", e); } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java index 0b71e1018..c74184c22 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java @@ -17,8 +17,11 @@ package org.apache.stormcrawler.opensearch.metrics; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.storm.Config; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -30,12 +33,8 @@ import org.apache.stormcrawler.opensearch.Constants; import org.apache.stormcrawler.opensearch.OpenSearchConnection; import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.core.CountRequest; -import org.opensearch.client.core.CountResponse; -import org.opensearch.core.action.ActionListener; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.FieldValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,45 +52,22 @@ public class StatusMetricsBolt extends BaseRichBolt { private String indexName; - private OpenSearchConnection connection; + private OpenSearchClient client; - private Map latestStatusCounts = new HashMap<>(6); + private Map latestStatusCounts = new ConcurrentHashMap<>(6); private int freqStats = 60; private OutputCollector _collector; - private transient StatusActionListener[] listeners; + private transient StatusCounter[] counters; - private class StatusActionListener implements ActionListener { + private static final class StatusCounter { + final String name; + final AtomicBoolean ready = new AtomicBoolean(true); - private final String name; - - private boolean ready = true; - - public boolean isReady() { - return ready; - } - - public void busy() { - this.ready = false; - } - - StatusActionListener(String statusName) { - name = statusName; - } - - @Override - public void onResponse(CountResponse response) { - ready = true; - LOG.debug("Got {} counts for status:{}", response.getCount(), name); - latestStatusCounts.put(name, response.getCount()); - } - - @Override - public void onFailure(Exception e) { - ready = true; - LOG.error("Failure when getting counts for status:{}", name, e); + StatusCounter(String name) { + this.name = name; } } @@ -101,7 +77,7 @@ public void prepare( _collector = collector; indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status"); try { - connection = OpenSearchConnection.getConnection(stormConf, OSBoltType); + client = OpenSearchConnection.getClient(stormConf, OSBoltType); } catch (Exception e1) { LOG.error("Can't connect to OpenSearch", e1); throw new RuntimeException(e1); @@ -110,14 +86,14 @@ public void prepare( CrawlerMetrics.registerGauge( context, stormConf, "status.count", () -> latestStatusCounts, freqStats); - listeners = new StatusActionListener[6]; + counters = new StatusCounter[6]; - listeners[0] = new StatusActionListener("DISCOVERED"); - listeners[1] = new StatusActionListener("FETCHED"); - listeners[2] = new StatusActionListener("FETCH_ERROR"); - listeners[3] = new StatusActionListener("REDIRECTION"); - listeners[4] = new StatusActionListener("ERROR"); - listeners[5] = new StatusActionListener("TOTAL"); + counters[0] = new StatusCounter("DISCOVERED"); + counters[1] = new StatusCounter("FETCHED"); + counters[2] = new StatusCounter("FETCH_ERROR"); + counters[3] = new StatusCounter("REDIRECTION"); + counters[4] = new StatusCounter("ERROR"); + counters[5] = new StatusCounter("TOTAL"); } @Override @@ -137,26 +113,69 @@ public void execute(Tuple input) { return; } - for (StatusActionListener listener : listeners) { + for (StatusCounter counter : counters) { // still waiting for results from previous request - if (!listener.isReady()) { - LOG.debug("Not ready to get counts for status {}", listener.name); + if (!counter.ready.compareAndSet(true, false)) { + LOG.debug("Not ready to get counts for status {}", counter.name); continue; } - CountRequest request = new CountRequest(indexName); - if (!listener.name.equalsIgnoreCase("TOTAL")) { - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.query(QueryBuilders.termQuery("status", listener.name)); - request.source(sourceBuilder); - } - listener.busy(); - connection.getClient().countAsync(request, RequestOptions.DEFAULT, listener); + final String statusName = counter.name; + CompletableFuture.supplyAsync( + () -> { + try { + if (statusName.equalsIgnoreCase("TOTAL")) { + return client.count(c -> c.index(indexName)); + } else { + return client.count( + c -> + c.index(indexName) + .query( + q -> + q.term( + t -> + t.field( + "status") + .value( + FieldValue + .of( + statusName))))); + } + } catch (Exception e) { + throw new CompletionException(e); + } + }) + .thenAccept( + response -> { + counter.ready.set(true); + LOG.debug( + "Got {} counts for status:{}", + response.count(), + statusName); + latestStatusCounts.put(statusName, response.count()); + }) + .exceptionally( + e -> { + counter.ready.set(true); + Throwable cause = + e instanceof CompletionException ? e.getCause() : e; + LOG.error( + "Failure when getting counts for status:{}", + statusName, + cause); + return null; + }); } } @Override public void cleanup() { - connection.close(); + if (client != null) { + try { + client._transport().close(); + } catch (Exception e) { + LOG.error("Exception closing client transport", e); + } + } } @Override diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java index d8995ff87..6f6d7be66 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java @@ -62,10 +62,7 @@ public class JSONResourceWrapper extends ParseFilter { public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) { refresher = new DelegateRefresher<>( - ParseFilter.class, - stormConf, - filterParams, - (delegate, conf, params) -> delegate.configure(conf, params)); + ParseFilter.class, stormConf, filterParams, ParseFilter::configure); } @Override @@ -75,6 +72,8 @@ public void filter(String url, byte[] content, DocumentFragment doc, ParseResult @Override public void cleanup() { - refresher.cleanup(); + if (refresher != null) { + refresher.cleanup(); + } } } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java index b29ebd167..0afe2b194 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpout.java @@ -19,10 +19,15 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.stormcrawler.Metadata; @@ -31,8 +36,8 @@ import org.apache.stormcrawler.opensearch.OpenSearchConnection; import org.apache.stormcrawler.persistence.AbstractQueryingSpout; import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.search.SearchHit; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch.OpenSearchClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,7 +77,7 @@ public abstract class AbstractSpout extends AbstractQueryingSpout { protected String indexName; - protected static RestHighLevelClient client; + protected static OpenSearchClient client; /** * when using multiple instances - each one is in charge of a specific shard useful when @@ -98,6 +103,15 @@ public abstract class AbstractSpout extends AbstractQueryingSpout { protected int queryTimeout = -1; + /** + * Single-threaded executor dedicated to running the asynchronous OpenSearch queries triggered + * from this spout. Using a dedicated, named, daemon executor avoids polluting the JVM-wide + * {@link java.util.concurrent.ForkJoinPool#commonPool() common pool} (which is shared with all + * other Storm tasks running in the same worker) and allows clean shutdown when the topology is + * stopped. + */ + protected ExecutorService queryExecutor; + @Override public void open( Map stormConf, @@ -106,6 +120,19 @@ public void open( super.open(stormConf, context, collector); + final String executorName = + "OpenSearchSpout-" + + context.getThisComponentId() + + "-" + + context.getThisTaskIndex(); + queryExecutor = + Executors.newSingleThreadExecutor( + r -> { + Thread t = new Thread(r, executorName); + t.setDaemon(true); + return t; + }); + indexName = ConfUtils.getString(stormConf, OSStatusIndexNameParamName, "status"); // one OS client per JVM @@ -180,14 +207,59 @@ public void open( /** Builds a query and use it retrieve the results from OS * */ protected abstract void populateBuffer(); - protected final boolean addHitToBuffer(SearchHit hit) { - Map keyValues = hit.getSourceAsMap(); - String url = (String) keyValues.get("url"); + /** + * Converts the {@link JsonData} carried by a search hit into a plain {@code Map} populated with native Java types (strings, numbers, lists, nested maps). + * + *

This helper exists to work around a subtle short-circuit inside {@code JsonDataImpl#to}: + * + * 

{@code
+     * if (clazz.isAssignableFrom(value.getClass())) {
+     *     return (T) value;
+     * }
+     * }
+ * + *

When {@link JsonData} is produced via {@code JsonData._DESERIALIZER} (as opposed to a raw + * Jackson {@code TreeNode}), its internal {@code value} is a Jakarta JSON-P {@code JsonObject} + * — typically a Parsson {@code JsonObjectImpl}. Crucially, {@code jakarta.json.JsonObject + * extends Map}, so calling {@code source.to(Map.class)} or {@code + * source.to(Object.class)} matches the {@code isAssignableFrom} guard and returns the raw + * {@code JsonObject} without ever consulting the configured {@link + * org.opensearch.client.json.jackson.JacksonJsonpMapper}. The resulting map values are + * still {@code JsonString}/{@code JsonStringImpl} instances and any downstream {@code (String) + * value} cast blows up with a {@link ClassCastException}. + * + *

Passing a concrete subclass that {@code JsonObject} does not implement (here + * {@link LinkedHashMap}) bypasses the short-circuit and forces the round-trip through {@code + * JacksonJsonpGenerator -> JacksonJsonpParser -> Jackson ObjectMapper.readValue}, which yields + * native Java values (String, Long, Double, ArrayList, LinkedHashMap…). + * + * @param source the document source carried by an OpenSearch hit; may be {@code null} + * @return a (possibly empty) map of native Java values, never {@code null} + */ + static Map sourceAsMap(JsonData source) { + if (source == null) { + return Collections.emptyMap(); + } + @SuppressWarnings("unchecked") + Map map = source.to(LinkedHashMap.class); + return map == null ? Collections.emptyMap() : map; + } + + /** + * Adds a document source to the URL buffer unless it is already being processed. + * + * @param source the document source as a key-value map (must contain a "url" entry) + * @return {@code true} if the URL was added to the buffer, {@code false} if it was already + * being processed or already present + */ + protected final boolean addHitToBuffer(Map source) { + String url = (String) source.get("url"); // is already being processed - skip it! if (beingProcessed.containsKey(url)) { return false; } - return buffer.add(url, fromKeyValues(keyValues)); + return buffer.add(url, fromKeyValues(source)); } protected final Metadata fromKeyValues(Map keyValues) { @@ -225,10 +297,29 @@ public void fail(Object msgId) { @Override public void close() { + // Shut the dedicated query executor down first so no more requests can be enqueued + // against the client while we are tearing it down. We give in-flight queries a short + // grace period to drain to keep shutdown reasonably graceful. + if (queryExecutor != null) { + queryExecutor.shutdown(); + try { + if (!queryExecutor.awaitTermination(30, TimeUnit.SECONDS)) { + LOG.warn( + "{} Query executor did not terminate within 30s, forcing shutdown", + logIdprefix); + queryExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + queryExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + } + queryExecutor = null; + } + synchronized (AbstractSpout.class) { if (client != null) { try { - client.close(); + client._transport().close(); } catch (IOException e) { LOG.error("Exception caught when closing client", e); } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java index 79936b364..a6272f368 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java @@ -17,45 +17,38 @@ package org.apache.stormcrawler.opensearch.persistence; -import static org.opensearch.index.query.QueryBuilders.boolQuery; - +import java.io.IOException; import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.Calendar; +import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TimeZone; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import org.apache.commons.lang3.StringUtils; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.opensearch.Constants; import org.apache.stormcrawler.util.ConfUtils; -import org.joda.time.format.ISODateTimeFormat; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.core.action.ActionListener; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.SearchHit; -import org.opensearch.search.aggregations.AggregationBuilders; -import org.opensearch.search.aggregations.Aggregations; -import org.opensearch.search.aggregations.BucketOrder; -import org.opensearch.search.aggregations.bucket.SingleBucketAggregation; -import org.opensearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder; -import org.opensearch.search.aggregations.bucket.terms.Terms; -import org.opensearch.search.aggregations.bucket.terms.Terms.Bucket; -import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.opensearch.search.aggregations.metrics.TopHits; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.sort.FieldSortBuilder; -import org.opensearch.search.sort.SortBuilders; -import org.opensearch.search.sort.SortOrder; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.SortOrder; +import org.opensearch.client.opensearch._types.aggregations.Aggregate; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.aggregations.StringTermsBucket; +import org.opensearch.client.opensearch._types.aggregations.TopHitsAggregate; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.search.Hit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +58,7 @@ * the same number of spout instances as OpenSearch shards. Guarantees a good mix of URLs by * aggregating them by an arbitrary field e.g. key. */ -public class AggregationSpout extends AbstractSpout implements ActionListener { +public class AggregationSpout extends AbstractSpout { private static final Logger LOG = LoggerFactory.getLogger(AggregationSpout.class); @@ -104,106 +97,171 @@ protected void populateBuffer() { lastTimeResetToNow = Instant.now(); } - String formattedQueryDate = ISODateTimeFormat.dateTimeNoMillis().print(queryDate.getTime()); + String formattedQueryDate = + Instant.ofEpochMilli(queryDate.getTime()) + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); LOG.info("{} Populating buffer with nextFetchDate <= {}", logIdprefix, formattedQueryDate); - BoolQueryBuilder queryBuilder = - boolQuery() - .filter(QueryBuilders.rangeQuery("nextFetchDate").lte(formattedQueryDate)); - - if (filterQueries != null) { - for (String filterQuery : filterQueries) { - queryBuilder.filter(QueryBuilders.queryStringQuery(filterQuery)); - } + // Build the top_hits sub-aggregation + Aggregation topHitsAgg = + Aggregation.of( + a -> + a.topHits( + th -> { + th.size(maxURLsPerBucket).explain(false); + for (String bsf : bucketSortField) { + th.sort( + s -> + s.field( + fs -> + fs.field(bsf) + .order( + SortOrder + .Asc))); + } + return th; + })); + + // Build the terms (partition) aggregation with top_hits sub-agg + Aggregation.Builder.ContainerBuilder partitionAggBuilder = + new Aggregation.Builder() + .terms( + t -> { + t.field(partitionField).size(maxBucketNum); + // sort between buckets by the min sub-aggregation + if (StringUtils.isNotBlank(totalSortField)) { + t.order( + Collections.singletonList( + Collections.singletonMap( + "top_hit", SortOrder.Asc))); + } + return t; + }) + .aggregations("docs", topHitsAgg); + + // add the min sub-aggregation used for sorting between buckets + if (StringUtils.isNotBlank(totalSortField)) { + partitionAggBuilder.aggregations( + "top_hit", Aggregation.of(minAgg -> minAgg.min(m -> m.field(totalSortField)))); } - SearchRequest request = new SearchRequest(indexName); - - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.query(queryBuilder); - sourceBuilder.from(0); - sourceBuilder.size(0); - sourceBuilder.explain(false); - sourceBuilder.trackTotalHits(false); + Aggregation partitionAgg = partitionAggBuilder.build(); + + // Build the search request + SearchRequest.Builder requestBuilder = + new SearchRequest.Builder() + .index(indexName) + .size(0) + .trackTotalHits(t -> t.enabled(false)) + .query( + q -> + q.bool( + b -> { + b.filter( + f -> + f.range( + r -> + r.field( + "nextFetchDate") + .lte( + JsonData + .of( + formattedQueryDate)))); + if (filterQueries != null) { + for (String fq : filterQueries) { + b.filter( + f -> + f.queryString( + qs -> + qs + .query( + fq))); + } + } + return b; + })); if (queryTimeout != -1) { - sourceBuilder.timeout( - new org.opensearch.common.unit.TimeValue(queryTimeout, TimeUnit.SECONDS)); - } - - TermsAggregationBuilder aggregations = - AggregationBuilders.terms("partition").field(partitionField).size(maxBucketNum); - - org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder tophits = - AggregationBuilders.topHits("docs").size(maxURLsPerBucket).explain(false); - - // sort within a bucket - for (String bsf : bucketSortField) { - FieldSortBuilder sorter = SortBuilders.fieldSort(bsf).order(SortOrder.ASC); - tophits.sort(sorter); - } - - aggregations.subAggregation(tophits); - - // sort between buckets - if (StringUtils.isNotBlank(totalSortField)) { - org.opensearch.search.aggregations.metrics.MinAggregationBuilder minBuilder = - AggregationBuilders.min("top_hit").field(totalSortField); - aggregations.subAggregation(minBuilder); - aggregations.order(BucketOrder.aggregation("top_hit", true)); + requestBuilder.timeout(queryTimeout + "s"); } if (sample) { - DiversifiedAggregationBuilder sab = new DiversifiedAggregationBuilder("sample"); - sab.field(partitionField).maxDocsPerValue(maxURLsPerBucket); - sab.shardSize(maxURLsPerBucket * maxBucketNum); - sab.subAggregation(aggregations); - sourceBuilder.aggregation(sab); + // Wrap in a diversified sampler aggregation + requestBuilder.aggregations( + "sample", + Aggregation.of( + a -> + a.diversifiedSampler( + ds -> + ds.field(partitionField) + .maxDocsPerValue( + maxURLsPerBucket) + .shardSize( + maxURLsPerBucket + * maxBucketNum)) + .aggregations("partition", partitionAgg))); } else { - sourceBuilder.aggregation(aggregations); + requestBuilder.aggregations("partition", partitionAgg); } - request.source(sourceBuilder); - - // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-preference.html - // _shards:2,3 - // specific shard but ideally a local copy of it + // shard preference for routing if (shardID != -1) { - request.preference("_shards:" + shardID + "|_local"); + requestBuilder.preference("_shards:" + shardID + "|_local"); } + SearchRequest request = requestBuilder.build(); + // dump query to log LOG.debug("{} OpenSearch query {}", logIdprefix, request); LOG.trace("{} isInQuery set to true", logIdprefix); isInQuery.set(true); - client.searchAsync(request, RequestOptions.DEFAULT, this); - } - @Override - public void onFailure(Exception arg0) { - LOG.error("{} Exception with OpenSearch query", logIdprefix, arg0); - markQueryReceivedNow(); + CompletableFuture.supplyAsync( + () -> { + try { + return client.search(request, JsonData.class); + } catch (IOException e) { + throw new CompletionException(e); + } + }, + queryExecutor) + .thenAccept(this::handleResponse) + .exceptionally( + e -> { + Throwable cause = e instanceof CompletionException ? e.getCause() : e; + LOG.error("{} Exception with OpenSearch query", logIdprefix, cause); + markQueryReceivedNow(); + return null; + }); } - @Override - public void onResponse(SearchResponse response) { + /** + * Handles the search response from an asynchronous aggregation query, extracting URLs from term + * buckets and adding them to the buffer. + * + * @param response the search response containing aggregation results + */ + protected void handleResponse(SearchResponse response) { long timeTaken = System.currentTimeMillis() - getTimeLastQuerySent(); - Aggregations aggregs = response.getAggregations(); + Map aggregs = response.aggregations(); - if (aggregs == null) { + if (aggregs == null || aggregs.isEmpty()) { markQueryReceivedNow(); return; } - SingleBucketAggregation sample = aggregs.get("sample"); - if (sample != null) { - aggregs = sample.getAggregations(); + // Unwrap the sample aggregation if present + Aggregate sampleAgg = aggregs.get("sample"); + if (sampleAgg != null) { + aggregs = sampleAgg.sampler().aggregations(); } - Terms agg = aggregs.get("partition"); + Aggregate partitionAgg = aggregs.get("partition"); + List buckets = partitionAgg.sterms().buckets().array(); int numhits = 0; int numBuckets = 0; @@ -214,35 +272,31 @@ public void onResponse(SearchResponse response) { currentBuckets.clear(); // For each entry - Iterator iterator = (Iterator) agg.getBuckets().iterator(); + Iterator iterator = buckets.iterator(); while (iterator.hasNext()) { - Terms.Bucket entry = iterator.next(); - String key = (String) entry.getKey(); // bucket key + StringTermsBucket entry = iterator.next(); + String key = entry.key(); // bucket key currentBuckets.add(key); - long docCount = entry.getDocCount(); // Doc count + long docCount = entry.docCount(); // Doc count int hitsForThisBucket = 0; - SearchHit lastHit = null; - + List lastSortValues = null; // filter results so that we don't include URLs we are already // being processed - TopHits topHits = entry.getAggregations().get("docs"); - for (SearchHit hit : topHits.getHits().getHits()) { + TopHitsAggregate topHits = entry.aggregations().get("docs").topHits(); + for (Hit hit : topHits.hits().hits()) { - LOG.debug( - "{} -> id [{}], _source [{}]", - logIdprefix, - hit.getId(), - hit.getSourceAsString()); + Map keyValues = sourceAsMap(hit.source()); + + LOG.debug("{} -> id [{}], _source [{}]", logIdprefix, hit.id(), keyValues); hitsForThisBucket++; - lastHit = hit; + lastSortValues = hit.sort(); - Map keyValues = hit.getSourceAsMap(); String url = (String) keyValues.get("url"); // consider only the first document of the last bucket @@ -273,8 +327,8 @@ public void onResponse(SearchResponse response) { LOG.debug("{} -> added to buffer : {}", logIdprefix, url); } - if (lastHit != null) { - sortValuesForKey(key, lastHit.getSortValues()); + if (lastSortValues != null && !lastSortValues.isEmpty()) { + sortValuesForKey(key, lastSortValues.toArray()); } if (hitsForThisBucket > 0) { @@ -369,5 +423,10 @@ public void onResponse(SearchResponse response) { markQueryReceivedNow(); } + /** + * Receives the sort values of the last hit returned for a bucket, as the {@link FieldValue} + * instances of the client. Subclasses can pass them to {@code search_after} to read the bucket + * from that position. + */ protected void sortValuesForKey(String key, Object[] sortValues) {} } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java index 551153f52..b3301e5c3 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/HybridSpout.java @@ -17,31 +17,28 @@ package org.apache.stormcrawler.opensearch.persistence; -import static org.opensearch.index.query.QueryBuilders.boolQuery; - import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import java.io.IOException; import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.stormcrawler.opensearch.Constants; import org.apache.stormcrawler.persistence.EmptyQueueListener; import org.apache.stormcrawler.util.ConfUtils; -import org.joda.time.format.ISODateTimeFormat; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.core.action.ActionListener; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.SearchHit; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.sort.FieldSortBuilder; -import org.opensearch.search.sort.SortBuilders; -import org.opensearch.search.sort.SortOrder; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.SortOrder; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.search.Hit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,7 +87,7 @@ public void emptyQueue(String queueName) { // would just overload OpenSearch and yield // mainly duplicates if (isInQuery.get()) { - LOG.trace("{} isInquery true", logIdprefix, queueName); + LOG.trace("{} isInquery true for {}", logIdprefix, queueName); return; } @@ -101,57 +98,89 @@ public void emptyQueue(String queueName) { lastTimeResetToNow = Instant.now(); } - String formattedQueryDate = ISODateTimeFormat.dateTimeNoMillis().print(queryDate.getTime()); - - BoolQueryBuilder queryBuilder = - boolQuery() - .filter(QueryBuilders.rangeQuery("nextFetchDate").lte(formattedQueryDate)); - - queryBuilder.filter(QueryBuilders.termQuery(partitionField, queueName)); - - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.query(queryBuilder); - sourceBuilder.from(0); - sourceBuilder.size(bufferReloadSize); - sourceBuilder.explain(false); - sourceBuilder.trackTotalHits(false); + String formattedQueryDate = + Instant.ofEpochMilli(queryDate.getTime()) + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + + SearchRequest.Builder requestBuilder = + new SearchRequest.Builder() + .index(indexName) + .size(bufferReloadSize) + .trackTotalHits(t -> t.enabled(false)) + .query( + q -> + q.bool( + b -> + b.filter( + f -> + f.range( + r -> + r.field( + "nextFetchDate") + .lte( + JsonData + .of( + formattedQueryDate)))) + .filter( + f -> + f.term( + t -> + t.field( + partitionField) + .value( + FieldValue + .of( + queueName)))))); // sort within a bucket for (String bsf : bucketSortField) { - FieldSortBuilder sorter = SortBuilders.fieldSort(bsf).order(SortOrder.ASC); - sourceBuilder.sort(sorter); + requestBuilder.sort(s -> s.field(fs -> fs.field(bsf).order(SortOrder.Asc))); } // do we have a search after for this one? Object[] searchAfterValues = searchAfterCache.getIfPresent(queueName); if (searchAfterValues != null) { - sourceBuilder.searchAfter(searchAfterValues); + for (Object sav : searchAfterValues) { + requestBuilder.searchAfter((FieldValue) sav); + } } - SearchRequest request = new SearchRequest(indexName); - - request.source(sourceBuilder); - - // https://www.elastic.co/guide/en/opensearch/reference/current/search-request-preference.html - // _shards:2,3 - // specific shard but ideally a local copy of it + // shard preference for routing if (shardID != -1) { - request.preference("_shards:" + shardID + "|_local"); + requestBuilder.preference("_shards:" + shardID + "|_local"); } - // dump query to log - LOG.debug("{} OpenSearch query {} - {}", logIdprefix, queueName, request.toString()); + SearchRequest request = requestBuilder.build(); - client.searchAsync(request, RequestOptions.DEFAULT, hrl); + // dump query to log + LOG.debug("{} OpenSearch query {} - {}", logIdprefix, queueName, request); + + CompletableFuture.supplyAsync( + () -> { + try { + return client.search(request, JsonData.class); + } catch (IOException e) { + throw new CompletionException(e); + } + }, + queryExecutor) + .thenAccept(hrl::handleResponse) + .exceptionally( + e -> { + Throwable cause = e instanceof CompletionException ? e.getCause() : e; + LOG.error("Exception with OpenSearch query", cause); + return null; + }); } /** Overrides the handling of responses for aggregations. */ @Override - public void onResponse(SearchResponse response) { + protected void handleResponse(SearchResponse response) { // delete all entries from the searchAfterCache when // we get the results from the aggregation spouts searchAfterCache.invalidateAll(); - super.onResponse(response); + super.handleResponse(response); } /** The aggregation kindly told us where to start from. */ @@ -163,40 +192,52 @@ protected void sortValuesForKey(String key, Object[] sortValues) { } /** Handling of results for a specific queue. */ - class HostResultListener implements ActionListener { + class HostResultListener { - @Override - public void onResponse(SearchResponse response) { + /** + * Handles the search response for a host-specific query, extracting hits and adding them to + * the buffer. + * + * @param response the search response containing document hits + */ + void handleResponse(SearchResponse response) { int alreadyprocessed = 0; int numDocs = 0; - SearchHit[] hits = response.getHits().getHits(); + List> hits = response.hits().hits(); Object[] sortValues = null; // retrieve the key for these results String key = null; - for (SearchHit hit : hits) { + for (Hit hit : hits) { numDocs++; + + Map source = sourceAsMap(hit.source()); + String pfield = partitionField; - Map sourceAsMap = hit.getSourceAsMap(); + Map fieldSource = source; if (pfield.startsWith("metadata.")) { - sourceAsMap = (Map) sourceAsMap.get("metadata"); + @SuppressWarnings("unchecked") + Map metadataMap = (Map) source.get("metadata"); + fieldSource = metadataMap; pfield = pfield.substring(9); } - Object key_as_object = sourceAsMap.get(pfield); + Object key_as_object = fieldSource.get(pfield); if (key_as_object instanceof List) { - if (((List) (key_as_object)).size() == 1) { - key = ((List) key_as_object).get(0); + @SuppressWarnings("unchecked") + List keyList = (List) key_as_object; + if (keyList.size() == 1) { + key = keyList.get(0); } } else { key = key_as_object.toString(); } - sortValues = hit.getSortValues(); - if (!addHitToBuffer(hit)) { + sortValues = hit.sort().toArray(); + if (!addHitToBuffer(source)) { alreadyprocessed++; } } @@ -214,14 +255,9 @@ public void onResponse(SearchResponse response) { "{} OpenSearch term query returned {} hits in {} msec with {} already being processed for {}", logIdprefix, numDocs, - response.getTook().getMillis(), + response.took(), alreadyprocessed, key); } - - @Override - public void onFailure(Exception e) { - LOG.error("Exception with OpenSearch query", e); - } } } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java index 3e1448721..6af7d9094 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/QueueBolt.java @@ -29,17 +29,15 @@ import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.tuple.Tuple; +import org.apache.stormcrawler.opensearch.AsyncBulkProcessor; import org.apache.stormcrawler.opensearch.Constants; import org.apache.stormcrawler.opensearch.IndexCreation; import org.apache.stormcrawler.opensearch.OpenSearchConnection; import org.apache.stormcrawler.util.ConfUtils; -import org.opensearch.action.DocWriteRequest; -import org.opensearch.action.bulk.BulkItemResponse; -import org.opensearch.action.bulk.BulkProcessor; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.core.rest.RestStatus; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +45,7 @@ * Sends information about the queues into an OpenSearch index. This has to be connected to a status * updater bolt. */ -public class QueueBolt extends BaseRichBolt implements BulkProcessor.Listener { +public class QueueBolt extends BaseRichBolt implements AsyncBulkProcessor.Listener { private static final Logger LOG = LoggerFactory.getLogger(QueueBolt.class); @@ -140,36 +138,39 @@ public void execute(Tuple tuple) { fields.put("key", key); fields.put("lastUpdated", Instant.now().toString()); - final IndexRequest indexRequest = - new IndexRequest(indexName).source(fields).id(docID).create(true); + final BulkOperation op = + BulkOperation.of(b -> b.create(c -> c.index(indexName).id(docID).document(fields))); - connection.addToProcessor(indexRequest); + connection.addToProcessor(op); } @Override public void beforeBulk(long executionId, BulkRequest request) { - LOG.debug("beforeBulk {} with {} actions", executionId, request.numberOfActions()); + LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size()); } @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - LOG.debug("afterBulk [{}] with {} responses", executionId, request.numberOfActions()); - for (BulkItemResponse item : response.getItems()) { - final String key = inFlight.remove(item.getId()); + LOG.debug("afterBulk [{}] with {} responses", executionId, response.items().size()); + for (BulkResponseItem item : response.items()) { + final String key = inFlight.remove(item.id()); if (key == null) { continue; } - if (!item.isFailed()) { + if (item.error() == null) { knownQueue.put(key, Boolean.TRUE); - } else if (RestStatus.CONFLICT.equals(item.getFailure().getStatus())) { + } else if (item.status() == 409) { // entry already exists in the index, e.g. written before a worker // restart or evicted from the cache - not an error knownQueue.put(key, Boolean.TRUE); } else { // do not cache the key so that the write is retried // when the key is next seen + final var error = item.error(); LOG.error( - "Failed to index queue entry {}: {}", key, item.getFailure().getMessage()); + "Failed to index queue entry {}: {}", + key, + error.reason() != null ? error.reason() : error.type()); } } } @@ -177,9 +178,9 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon @Override public void afterBulk(long executionId, BulkRequest request, Throwable failure) { LOG.error("Exception with bulk {} - failing the whole lot ", executionId, failure); - for (DocWriteRequest docWriteRequest : request.requests()) { + for (BulkOperation op : request.operations()) { // not cached - the writes are retried when the keys are next seen - inFlight.remove(docWriteRequest.id()); + inFlight.remove(OpenSearchConnection.getBulkOperationId(op)); } } diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java index f025f582c..2e0b92eb8 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java @@ -18,7 +18,9 @@ package org.apache.stormcrawler.opensearch.persistence; import java.io.IOException; +import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -31,6 +33,7 @@ import org.apache.stormcrawler.metrics.CrawlerMetrics; import org.apache.stormcrawler.metrics.ScopedCounter; import org.apache.stormcrawler.metrics.ScopedReducedMetric; +import org.apache.stormcrawler.opensearch.AsyncBulkProcessor; import org.apache.stormcrawler.opensearch.Constants; import org.apache.stormcrawler.opensearch.IndexCreation; import org.apache.stormcrawler.opensearch.OpenSearchConnection; @@ -39,12 +42,9 @@ import org.apache.stormcrawler.persistence.Status; import org.apache.stormcrawler.util.ConfUtils; import org.apache.stormcrawler.util.URLPartitioner; -import org.opensearch.action.bulk.BulkProcessor; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,7 +52,8 @@ * Simple bolt which stores the status of URLs into OpenSearch. Takes the tuples coming from the * 'status' stream. To be used in combination with a Spout to read from the index. */ -public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt implements BulkProcessor.Listener { +public class StatusUpdaterBolt extends AbstractStatusUpdaterBolt + implements AsyncBulkProcessor.Listener { private static final Logger LOG = LoggerFactory.getLogger(StatusUpdaterBolt.class); @@ -142,6 +143,16 @@ public void prepare( int metrics_time_bucket_secs = 30; + // benchmarking - average number of items received back from OpenSearch per second + this.receivedPerSecMetrics = + CrawlerMetrics.registerPerSecMetric( + context, stormConf, "average_persec", metrics_time_bucket_secs); + + // eventCounter MUST be registered before WaitAckCache — the eviction lambda captures it + this.eventCounter = + CrawlerMetrics.registerCounter( + context, stormConf, "counters", metrics_time_bucket_secs); + String defaultSpec = String.format( Locale.ROOT, @@ -159,24 +170,13 @@ public void prepare( eventCounter.scope("purged").incrBy(1); collector.fail(t); }); - - // create gauge for waitAck CrawlerMetrics.registerGauge( context, stormConf, "waitAck", waitAck::estimatedSize, metrics_time_bucket_secs); - // benchmarking - average number of items received back by Elastic per second - this.receivedPerSecMetrics = - CrawlerMetrics.registerPerSecMetric( - context, stormConf, "average_persec", metrics_time_bucket_secs); - - this.eventCounter = - CrawlerMetrics.registerCounter( - context, stormConf, "counters", metrics_time_bucket_secs); - try { connection = OpenSearchConnection.getConnection(stormConf, OSBoltType, this); } catch (Exception e1) { - LOG.error("Can't connect to ElasticSearch", e1); + LOG.error("Can't connect to OpenSearch", e1); throw new RuntimeException(e1); } @@ -205,8 +205,10 @@ public void store( String documentID = getDocumentID(metadata, url); - // check that the same URL is not being sent to OpenSearch - if (status.equals(Status.DISCOVERED) && waitAck.contains(documentID)) { + boolean isAlreadySentAndDiscovered = + status.equals(Status.DISCOVERED) && waitAck.contains(documentID); + + if (isAlreadySentAndDiscovered) { // if this object is discovered - adding another version of it // won't make any difference LOG.debug( @@ -220,16 +222,16 @@ public void store( return; } - XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - builder.field("url", url); - builder.field("status", status); + Map doc = new HashMap<>(); + doc.put("url", url); + doc.put("status", status.name()); - builder.startObject("metadata"); + Map metadataMap = new HashMap<>(); for (String mdKey : metadata.keySet()) { String[] values = metadata.getValues(mdKey); // periods are not allowed - replace with %2E mdKey = mdKey.replaceAll("\\.", "%2E"); - builder.array(mdKey, values); + metadataMap.put(mdKey, Arrays.asList(values)); } String partitionKey = partitioner.getPartition(url, metadata); @@ -245,62 +247,87 @@ public void store( // store routing key in metadata? if (StringUtils.isNotBlank(fieldNameForRoutingKey) && routingFieldNameInMetadata) { - builder.field(fieldNameForRoutingKey, partitionKey); + metadataMap.put(fieldNameForRoutingKey, partitionKey); } - builder.endObject(); + doc.put("metadata", metadataMap); // store routing key outside metadata? if (StringUtils.isNotBlank(fieldNameForRoutingKey) && !routingFieldNameInMetadata) { - builder.field(fieldNameForRoutingKey, partitionKey); + doc.put(fieldNameForRoutingKey, partitionKey); } if (nextFetch.isPresent()) { - builder.timeField("nextFetchDate", nextFetch.get()); + doc.put("nextFetchDate", nextFetch.get().toInstant().toString()); } - - builder.endObject(); - - IndexRequest request = new IndexRequest(getIndexName(metadata)); - // check that we don't overwrite an existing entry // When create is used, the index operation will fail if a document // by that id already exists in the index. final boolean create = status.equals(Status.DISCOVERED); - request.source(builder).id(documentID).create(create); - - if (doRouting) { - request.routing(partitionKey); + final String targetIndex = getIndexName(metadata); + final String routing = doRouting ? partitionKey : null; + + BulkOperation op; + if (create) { + op = + BulkOperation.of( + b -> + b.create( + c -> { + c.index(targetIndex).id(documentID).document(doc); + if (routing != null) { + c.routing(routing); + } + return c; + })); + } else { + op = + BulkOperation.of( + b -> + b.index( + idx -> { + idx.index(targetIndex).id(documentID).document(doc); + if (routing != null) { + idx.routing(routing); + } + return idx; + })); } waitAck.addTuple(documentID, tuple); LOG.debug("Sending to OpenSearch buffer {} with ID {}", url, documentID); - connection.addToProcessor(request); + try { + connection.addToProcessor(op); + } catch (Exception e) { + // the caller fails the tuple, so no bulk response may ack or fail it again + waitAck.invalidate(documentID); + throw e; + } } @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - LOG.debug("afterBulk [{}] with {} responses", executionId, request.numberOfActions()); + LOG.debug("afterBulk [{}] with {} responses", executionId, request.operations().size()); eventCounter.scope("bulks_received").incrBy(1); - eventCounter.scope("bulk_msec").incrBy(response.getTook().getMillis()); - eventCounter.scope("received").incrBy(request.numberOfActions()); - receivedPerSecMetrics.scope("received").update(request.numberOfActions()); + eventCounter.scope("bulk_msec").incrBy(response.took()); + eventCounter.scope("received").incrBy(request.operations().size()); + receivedPerSecMetrics.scope("received").update(request.operations().size()); waitAck.processBulkResponse( response, executionId, eventCounter, - (id, t, selected) -> { - if (!selected.failed) { - String url = t.getStringByField("url"); + (id, tuple, selected) -> { + if (!selected.failed()) { + String url = tuple.getStringByField("url"); LOG.debug("Acked {} with ID {}", url, id); eventCounter.scope("acked").incrBy(1); - ack(t, url); + super.ack(tuple, url); } else { eventCounter.scope("failed").incrBy(1); - collector.fail(t); + collector.fail(tuple); } }); } @@ -308,8 +335,8 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon @Override public void afterBulk(long executionId, BulkRequest request, Throwable throwable) { eventCounter.scope("bulks_received").incrBy(1); - eventCounter.scope("received").incrBy(request.numberOfActions()); - receivedPerSecMetrics.scope("received").update(request.numberOfActions()); + eventCounter.scope("received").incrBy(request.operations().size()); + receivedPerSecMetrics.scope("received").update(request.operations().size()); waitAck.processFailedBulk( request, @@ -323,7 +350,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable throwable @Override public void beforeBulk(long executionId, BulkRequest request) { - LOG.debug("beforeBulk {} with {} actions", executionId, request.numberOfActions()); + LOG.debug("beforeBulk {} with {} actions", executionId, request.operations().size()); eventCounter.scope("bulks_sent").incrBy(1); } diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java similarity index 100% rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorHttpTest.java diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java similarity index 100% rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorRetryTest.java diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java similarity index 100% rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java index ae90bebde..bb484e843 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java @@ -23,186 +23,95 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.UncheckedIOException; -import java.net.InetAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.Credentials; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.core5.http.HttpHost; import org.junit.jupiter.api.Test; -import org.opensearch.client.Request; -import org.opensearch.client.RestClient; -import org.opensearch.client.sniff.OpenSearchNodesSniffer; class OpenSearchConnectionCredentialsTest { - private static final Credentials CREDENTIALS = - new UsernamePasswordCredentials("crawler", "s3cret"); - - private static CredentialsProvider providerFor(HttpHost... hosts) { - return new OpenSearchConnection.OriginCredentialsProvider(List.of(hosts), CREDENTIALS); - } - - /** The scope HttpClient looks the credentials up with for a request to the given node. */ - private static AuthScope request(String scheme, String host, int port) { - return new AuthScope(new HttpHost(host, port, scheme), AuthScope.ANY_REALM, "Basic"); + private static BasicCredentialsProvider providerFor(List hosts) { + final BasicCredentialsProvider provider = new BasicCredentialsProvider(); + final UsernamePasswordCredentials credentials = + new UsernamePasswordCredentials("crawler", "s3cret".toCharArray()); + for (AuthScope scope : OpenSearchConnection.credentialScopes(hosts)) { + provider.setCredentials(scope, credentials); + } + return provider; } @Test - void credentialsAreGivenToTheConfiguredAddresses() { - final CredentialsProvider provider = - providerFor( - new HttpHost("opensearch1.example.org", 9200, "https"), - new HttpHost("OpenSearch2.example.org", 9201, "https")); - assertEquals( - CREDENTIALS, - provider.getCredentials(request("https", "opensearch1.example.org", 9200))); + void scopesCoverTheConfiguredHostsAndPorts() { + final List scopes = + OpenSearchConnection.credentialScopes( + List.of( + new HttpHost("https", "opensearch1.example.org", 9200), + new HttpHost("https", "opensearch2.example.org", 9201))); assertEquals( - CREDENTIALS, - provider.getCredentials(request("HTTPS", "opensearch2.example.org", 9201))); + List.of( + new AuthScope("https", "opensearch1.example.org", 9200, null, null), + new AuthScope("https", "opensearch2.example.org", 9201, null, null)), + scopes); } @Test - void credentialsAreOnlyGivenToTheConfiguredHosts() { - final CredentialsProvider provider = - providerFor(new HttpHost("opensearch1.example.org", 9200, "https")); - - // a node found by the sniffer under the address it publishes - assertNull(provider.getCredentials(request("https", "10.0.0.12", 9200))); - // same host, another port - assertNull(provider.getCredentials(request("https", "opensearch1.example.org", 9300))); - assertNull(provider.getCredentials(request("https", "other.example.org", 9200))); - } - - @Test - void credentialsAreMatchedOnTheScheme() { - final CredentialsProvider provider = - providerFor(new HttpHost("opensearch1.example.org", 9200, "https")); - assertNull(provider.getCredentials(request("http", "opensearch1.example.org", 9200))); + void duplicateAddressesGiveOneScope() { + final List scopes = + OpenSearchConnection.credentialScopes( + List.of( + new HttpHost("https", "opensearch1.example.org", 9200), + new HttpHost("https", "OpenSearch1.example.org", 9200))); + assertEquals(1, scopes.size()); } @Test - void credentialsNeedAnOrigin() { - final CredentialsProvider provider = - providerFor(new HttpHost("opensearch1.example.org", 9200, "https")); - assertNull(provider.getCredentials(new AuthScope("opensearch1.example.org", 9200))); - assertNull(provider.getCredentials(AuthScope.ANY)); + void noAddressGivesNoScope() { + assertTrue(OpenSearchConnection.credentialScopes(List.of()).isEmpty()); } @Test - void noAddressGivesTheCredentialsToNoOne() { - assertNull(providerFor().getCredentials(request("https", "opensearch1.example.org", 9200))); - } + void credentialsAreOnlyGivenToTheConfiguredHosts() { + final BasicCredentialsProvider provider = + providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200))); - /** - * Sends a request with the client to a local server and returns the Authorization header it - * received, if any. - */ - private static String authorizationSentTo(HttpHost configured) throws Exception { - try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) { - final int port = server.getLocalPort(); - final CompletableFuture authorization = - CompletableFuture.supplyAsync( - () -> { - try (Socket socket = server.accept()) { - final BufferedReader in = - new BufferedReader( - new InputStreamReader( - socket.getInputStream(), - StandardCharsets.US_ASCII)); - String header = null; - for (String line = in.readLine(); - line != null && !line.isEmpty(); - line = in.readLine()) { - if (line.regionMatches(true, 0, "Authorization:", 0, 14)) { - header = line.substring(14).trim(); - } - } - socket.getOutputStream() - .write( - "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" - .getBytes(StandardCharsets.US_ASCII)); - return header; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - final HttpHost configuredHere = - new HttpHost(configured.getHostName(), port, configured.getSchemeName()); - try (RestClient client = - RestClient.builder(new HttpHost("127.0.0.1", port, "http")) - .setHttpClientConfigCallback( - b -> - b.setDefaultCredentialsProvider( - providerFor(configuredHere))) - .build()) { - client.performRequest(new Request("GET", "/")); - } - return authorization.get(10, TimeUnit.SECONDS); - } + assertNotNull( + provider.getCredentials(request("https", "opensearch1.example.org", 9200), null)); + // another node of the cluster which is not listed in the addresses + assertNull(provider.getCredentials(request("https", "10.0.0.12", 9200), null)); + // same host, another port + assertNull( + provider.getCredentials(request("https", "opensearch1.example.org", 9300), null)); + assertNull(provider.getCredentials(request("https", "other.example.org", 9200), null)); } @Test - void clientSendsTheCredentialsToAConfiguredAddress() throws Exception { - assertNotNull(authorizationSentTo(new HttpHost("127.0.0.1", 0, "http"))); + void credentialsAreMatchedOnTheScheme() { + final BasicCredentialsProvider provider = + providerFor(List.of(new HttpHost("https", "opensearch1.example.org", 9200))); + assertNull(provider.getCredentials(request("http", "opensearch1.example.org", 9200), null)); } - @Test - void clientSendsNoCredentialsToAnotherAddress() throws Exception { - assertNull(authorizationSentTo(new HttpHost("127.0.0.1", 0, "https"))); + /** The scope HttpClient looks the credentials up with for a request to the given node. */ + private static AuthScope request(String scheme, String host, int port) { + return new AuthScope(new HttpHost(scheme, host, port), null, "Basic"); } @Test void plainHttpToARemoteHostIsReported() { - final HttpHost remote = new HttpHost("opensearch1.example.org", 9200, "http"); + final HttpHost remote = new HttpHost("http", "opensearch1.example.org", 9200); final List plain = OpenSearchConnection.plainHttpHosts( List.of( remote, - new HttpHost("opensearch2.example.org", 9200, "https"), - new HttpHost("localhost", 9200, "http"), - new HttpHost("127.0.0.1", 9200, "http"), - new HttpHost("[::1]", 9200, "http"))); + new HttpHost("https", "opensearch2.example.org", 9200), + new HttpHost("http", "localhost", 9200), + new HttpHost("http", "127.0.0.1", 9200), + new HttpHost("http", "[::1]", 9200))); assertEquals(List.of(remote), plain); } - @Test - void sniffedNodesKeepTheSchemeOfTheAddresses() { - assertEquals( - OpenSearchNodesSniffer.Scheme.HTTPS, - OpenSearchConnection.sniffScheme( - List.of(new HttpHost("opensearch1.example.org", 9200, "https")))); - assertEquals( - OpenSearchNodesSniffer.Scheme.HTTPS, - OpenSearchConnection.sniffScheme( - List.of( - new HttpHost("opensearch1.example.org", 9200, "http"), - new HttpHost("opensearch2.example.org", 9200, "https")))); - assertEquals( - OpenSearchNodesSniffer.Scheme.HTTP, - OpenSearchConnection.sniffScheme( - List.of(new HttpHost("opensearch1.example.org", 9200, "http")))); - } - - @Test - void credentialsNeedUserAndPassword() { - assertTrue(OpenSearchConnection.hasCredentials("crawler", "s3cret")); - assertFalse(OpenSearchConnection.hasCredentials("crawler", null)); - assertFalse(OpenSearchConnection.hasCredentials("crawler", " ")); - assertFalse(OpenSearchConnection.hasCredentials(null, "s3cret")); - assertFalse(OpenSearchConnection.hasCredentials("", "")); - } - @Test void loopbackAddresses() { assertTrue(OpenSearchConnection.isLoopback("localhost")); diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java similarity index 100% rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java index 0fd22d181..3bd2dc6d4 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java @@ -17,31 +17,29 @@ package org.apache.stormcrawler.opensearch; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.github.benmanes.caffeine.cache.Ticker; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.metrics.ScopedCounter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.opensearch.action.DocWriteRequest; -import org.opensearch.action.bulk.BulkItemResponse; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.bulk.BulkResponse; -import org.opensearch.action.delete.DeleteRequest; -import org.opensearch.action.index.IndexResponse; -import org.opensearch.core.index.shard.ShardId; -import org.opensearch.core.rest.RestStatus; +import org.opensearch.client.opensearch._types.ErrorCause; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; +import org.opensearch.client.opensearch.core.bulk.OperationType; import org.slf4j.LoggerFactory; class WaitAckCacheTest { @@ -66,23 +64,33 @@ private Tuple mockTuple(String url) { return t; } - private static ShardId shardId() { - return new ShardId("index", "_na_", 0); + private static BulkResponseItem successItem(String docId) { + return BulkResponseItem.of( + b -> b.id(docId).index("index").status(200).operationType(OperationType.Index)); } - private static BulkItemResponse successItem(int itemId, String docId) { - IndexResponse indexResponse = new IndexResponse(shardId(), docId, 1, 1, 1, true); - return new BulkItemResponse(itemId, DocWriteRequest.OpType.INDEX, indexResponse); + private static BulkResponseItem failedItem(String docId, int status) { + return BulkResponseItem.of( + b -> + b.id(docId) + .index("index") + .status(status) + .operationType(OperationType.Index) + .error( + ErrorCause.of( + e -> e.type("test_error").reason("test failure")))); } - private static BulkItemResponse failedItem(int itemId, String docId, RestStatus status) { - BulkItemResponse.Failure failure = - new BulkItemResponse.Failure("index", docId, new Exception("test failure"), status); - return new BulkItemResponse(itemId, DocWriteRequest.OpType.INDEX, failure); - } - - private static BulkResponse bulkResponse(BulkItemResponse... items) { - return new BulkResponse(items, 10L); + private static BulkResponse bulkResponse(BulkResponseItem... items) { + boolean hasErrors = false; + for (BulkResponseItem item : items) { + if (item.error() != null) { + hasErrors = true; + break; + } + } + final boolean errors = hasErrors; + return BulkResponse.of(b -> b.took(10).errors(errors).items(List.of(items))); } @Test @@ -110,14 +118,14 @@ void processBulkResponse_successfulItem_ackedViaTupleAction() { Tuple t = mockTuple("http://example.com"); cache.addTuple("doc1", t); - BulkResponse response = bulkResponse(successItem(0, "doc1")); + BulkResponse response = bulkResponse(successItem("doc1")); cache.processBulkResponse( response, 1L, null, (id, tuple, selected) -> { - if (!selected.failed) { + if (!selected.failed()) { acked.add(tuple); } else { failed.add(tuple); @@ -135,15 +143,14 @@ void processBulkResponse_failedItem_failedViaTupleAction() { Tuple t = mockTuple("http://example.com"); cache.addTuple("doc1", t); - BulkResponse response = - bulkResponse(failedItem(0, "doc1", RestStatus.INTERNAL_SERVER_ERROR)); + BulkResponse response = bulkResponse(failedItem("doc1", 500)); cache.processBulkResponse( response, 1L, null, (id, tuple, selected) -> { - if (!selected.failed) { + if (!selected.failed()) { acked.add(tuple); } else { failed.add(tuple); @@ -160,15 +167,18 @@ void processBulkResponse_conflictIsNotAFailure() { Tuple t = mockTuple("http://example.com"); cache.addTuple("doc1", t); - ScopedCounter counter = scopeName -> incrementBy -> {}; - BulkResponse response = bulkResponse(failedItem(0, "doc1", RestStatus.CONFLICT)); + ScopedCounter counter = mock(ScopedCounter.class); + ScopedCounter.CountHandle handle = mock(ScopedCounter.CountHandle.class); + when(counter.scope("doc_conflicts")).thenReturn(handle); + + BulkResponse response = bulkResponse(failedItem("doc1", 409)); cache.processBulkResponse( response, 1L, counter, (id, tuple, selected) -> { - if (!selected.failed) { + if (!selected.failed()) { acked.add(tuple); } else { failed.add(tuple); @@ -177,6 +187,7 @@ void processBulkResponse_conflictIsNotAFailure() { assertEquals(1, acked.size()); assertEquals(0, failed.size()); + verify(handle).incrBy(1); } @Test @@ -186,7 +197,7 @@ void processBulkResponse_multipleTuplesForSameDocId() { cache.addTuple("doc1", t1); cache.addTuple("doc1", t2); - BulkResponse response = bulkResponse(successItem(0, "doc1")); + BulkResponse response = bulkResponse(successItem("doc1")); cache.processBulkResponse(response, 1L, null, (id, tuple, selected) -> acked.add(tuple)); @@ -201,17 +212,14 @@ void processBulkResponse_duplicateDocIdInBulk_prefersSuccess() { Tuple t = mockTuple("http://example.com"); cache.addTuple("doc1", t); - BulkResponse response = - bulkResponse( - failedItem(0, "doc1", RestStatus.INTERNAL_SERVER_ERROR), - successItem(1, "doc1")); + BulkResponse response = bulkResponse(failedItem("doc1", 500), successItem("doc1")); cache.processBulkResponse( response, 1L, null, (id, tuple, selected) -> { - if (!selected.failed) { + if (!selected.failed()) { acked.add(tuple); } else { failed.add(tuple); @@ -229,9 +237,14 @@ void processFailedBulk_failsAllMatchingTuples() { cache.addTuple("doc1", t1); cache.addTuple("doc2", t2); - BulkRequest request = new BulkRequest(); - request.add(new DeleteRequest("index", "doc1")); - request.add(new DeleteRequest("index", "doc2")); + BulkRequest request = + BulkRequest.of( + b -> + b.operations( + BulkOperation.of( + o -> o.delete(d -> d.index("index").id("doc1"))), + BulkOperation.of( + o -> o.delete(d -> d.index("index").id("doc2"))))); cache.processFailedBulk(request, 1L, new Exception("connection lost"), failed::add); @@ -247,8 +260,17 @@ void processFailedBulk_ignoresMissingIds() { Tuple t = mockTuple("http://example.com"); cache.addTuple("doc1", t); - BulkRequest request = new BulkRequest(); - request.add(new DeleteRequest("index", "doc_unknown")); + BulkRequest request = + BulkRequest.of( + b -> + b.operations( + BulkOperation.of( + o -> + o.delete( + d -> + d.index("index") + .id( + "doc_unknown"))))); cache.processFailedBulk(request, 1L, new Exception("test"), failed::add); @@ -259,25 +281,29 @@ void processFailedBulk_ignoresMissingIds() { @Test void eviction_failsTuplesOnExpiry() { - AtomicLong fakeTime = new AtomicLong(0); - Ticker fakeTicker = fakeTime::get; cache = new WaitAckCache( "expireAfterWrite=1s", LoggerFactory.getLogger(WaitAckCacheTest.class), - evicted::add, - fakeTicker); + evicted::add); Tuple t = mockTuple("http://example.com"); cache.addTuple("doc1", t); - // Advance past the 1s expiry - fakeTime.set(TimeUnit.SECONDS.toNanos(2)); - - // Access triggers expiration; cleanUp forces listener execution - cache.contains("doc1"); - cache.cleanUp(); + // Force cache maintenance after expiry by doing a contains() check + // which accesses the cache and triggers Caffeine's cleanup + await().atMost(5, TimeUnit.SECONDS) + .pollInterval(200, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + // contains() accesses the cache which triggers cleanup + cache.contains("doc1"); + // also try adding and invalidating a dummy entry to force maintenance + Tuple dummy = mockTuple("http://dummy"); + cache.addTuple("_probe_", dummy); + cache.invalidate("_probe_"); + assertFalse(evicted.isEmpty(), "Eviction callback should have fired"); + }); - assertFalse(evicted.isEmpty(), "Eviction callback should have fired"); assertTrue(evicted.contains(t)); } @@ -288,17 +314,14 @@ void processBulkResponse_multipleDocIds() { cache.addTuple("doc1", t1); cache.addTuple("doc2", t2); - BulkResponse response = - bulkResponse( - successItem(0, "doc1"), - failedItem(1, "doc2", RestStatus.INTERNAL_SERVER_ERROR)); + BulkResponse response = bulkResponse(successItem("doc1"), failedItem("doc2", 500)); cache.processBulkResponse( response, 1L, null, (id, tuple, selected) -> { - if (!selected.failed) { + if (!selected.failed()) { acked.add(tuple); } else { failed.add(tuple); @@ -310,20 +333,4 @@ void processBulkResponse_multipleDocIds() { assertEquals(1, failed.size()); assertSame(t2, failed.get(0)); } - - @Test - void shutdown_failsAllRemainingTuples() { - Tuple t1 = mockTuple("http://example.com/1"); - Tuple t2 = mockTuple("http://example.com/2"); - cache.addTuple("doc1", t1); - cache.addTuple("doc2", t2); - - cache.shutdown(); - - assertEquals(2, evicted.size()); - assertTrue(evicted.contains(t1)); - assertTrue(evicted.contains(t2)); - assertFalse(cache.contains("doc1")); - assertFalse(cache.contains("doc2")); - } } diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java index e9c72b336..44150ea67 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java @@ -26,7 +26,8 @@ @Testcontainers(disabledWithoutDocker = true) public abstract class AbstractOpenSearchTest { - private static final String OPENSEARCH_VERSION = "2.19.4"; + private static final String OPENSEARCH_VERSION = + System.getProperty("opensearch-version", "2.19.5"); public static final String PASSWORD = "This1sAPassw0rd"; diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java similarity index 100% rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/NullMetadataValueTest.java diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java index 92b665c5f..296c58dd5 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/QueueBoltTest.java @@ -31,7 +31,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.http.HttpHost; +import org.apache.hc.core5.http.HttpHost; import org.apache.storm.task.OutputCollector; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Metadata; @@ -44,12 +44,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.opensearch.action.get.GetRequest; -import org.opensearch.action.get.GetResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestClient; -import org.opensearch.client.RestClientBuilder; -import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.GetResponse; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,7 +58,9 @@ class QueueBoltTest extends AbstractOpenSearchTest { protected TestOutputCollector output; - protected org.opensearch.client.RestHighLevelClient client; + protected OpenSearchClient client; + + private OpenSearchTransport transport; private static final Logger LOG = LoggerFactory.getLogger(QueueBoltTest.class); @@ -78,12 +79,15 @@ static void afterClass() { @BeforeEach void setupQueueBolt() throws IOException { - RestClientBuilder builder = - RestClient.builder( - new HttpHost( - opensearchContainer.getHost(), - opensearchContainer.getMappedPort(9200))); - client = new RestHighLevelClient(builder); + transport = + ApacheHttpClient5TransportBuilder.builder( + new HttpHost( + "http", + opensearchContainer.getHost(), + opensearchContainer.getMappedPort(9200))) + .setMapper(new JacksonJsonpMapper()) + .build(); + client = new OpenSearchClient(transport); output = new TestOutputCollector(); bolt = prepareQueueBolt(); } @@ -105,7 +109,7 @@ void close() { bolt.cleanup(); output = null; try { - client.close(); + transport.close(); } catch (IOException e) { } } @@ -129,20 +133,19 @@ private void awaitIndexed(String key) { .until( () -> { try { - GetResponse result = - client.get( - new GetRequest("queues", id), - RequestOptions.DEFAULT); - return result.isExists(); + GetResponse result = + client.get(g -> g.index("queues").id(id), Map.class); + return result.found(); } catch (IOException e) { return false; } }); } + @SuppressWarnings("unchecked") private Map getSource(String key) throws IOException { String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(key); - return client.get(new GetRequest("queues", id), RequestOptions.DEFAULT).getSourceAsMap(); + return client.get(g -> g.index("queues").id(id), Map.class).source(); } @Test diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java index 19df14538..71fe77957 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/StatusBoltTest.java @@ -32,7 +32,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.http.HttpHost; +import org.apache.hc.core5.http.HttpHost; import org.apache.storm.task.OutputCollector; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Metadata; @@ -46,12 +46,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.opensearch.action.get.GetRequest; -import org.opensearch.action.get.GetResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestClient; -import org.opensearch.client.RestClientBuilder; -import org.opensearch.client.RestHighLevelClient; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.GetResponse; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,7 +60,9 @@ class StatusBoltTest extends AbstractOpenSearchTest { protected TestOutputCollector output; - protected org.opensearch.client.RestHighLevelClient client; + protected OpenSearchClient client; + + private OpenSearchTransport transport; private static final Logger LOG = LoggerFactory.getLogger(StatusBoltTest.class); @@ -81,12 +82,15 @@ static void afterClass() { @BeforeEach void setupStatusBolt() throws IOException { bolt = new StatusUpdaterBolt(); - RestClientBuilder builder = - RestClient.builder( - new HttpHost( - opensearchContainer.getHost(), - opensearchContainer.getMappedPort(9200))); - client = new RestHighLevelClient(builder); + transport = + ApacheHttpClient5TransportBuilder.builder( + new HttpHost( + "http", + opensearchContainer.getHost(), + opensearchContainer.getMappedPort(9200))) + .setMapper(new JacksonJsonpMapper()) + .build(); + client = new OpenSearchClient(transport); // configure the status updater bolt Map conf = new HashMap<>(); conf.put("opensearch.status.routing.fieldname", "metadata.key"); @@ -107,7 +111,7 @@ void close() { bolt.cleanup(); output = null; try { - client.close(); + transport.close(); } catch (IOException e) { } } @@ -129,6 +133,7 @@ private Future store(String url, Status status, Metadata metadata) { @Test @Timeout(value = 2, unit = TimeUnit.MINUTES) // see https://github.com/apache/stormcrawler/issues/885 + @SuppressWarnings("unchecked") void checkListKeyFromOpensearch() throws IOException, ExecutionException, InterruptedException, TimeoutException { String url = "https://www.url.net/something"; @@ -136,10 +141,10 @@ void checkListKeyFromOpensearch() md.addValue("someKey", "someValue"); store(url, Status.DISCOVERED, md).get(10, TimeUnit.SECONDS); assertEquals(1, output.getAckedTuples().size()); - // check output in Opensearch? + // check output in Opensearch String id = org.apache.commons.codec.digest.DigestUtils.sha256Hex(url); - GetResponse result = client.get(new GetRequest("status", id), RequestOptions.DEFAULT); - Map sourceAsMap = result.getSourceAsMap(); + GetResponse result = client.get(g -> g.index("status").id(id), Map.class); + Map sourceAsMap = result.source(); final String pfield = "metadata.somekey"; sourceAsMap = (Map) sourceAsMap.get("metadata"); final var pfieldNew = pfield.substring(9); diff --git a/external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java similarity index 100% rename from external/opensearch-java/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java rename to external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java From 848afc340f9c9d213308090109f949b78a6f9046 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Fri, 16 Oct 2026 15:56:12 +0200 Subject: [PATCH 03/14] build external/opensearch with the opensearch-java client The module depends on opensearch-java 3.10.0 instead of the high-level REST client and the sniffer, and its tests run against OpenSearch 3.5.0. The artifactId stays stormcrawler-opensearch. external/opensearch-java leaves the reactor; its README is merged into the one of external/opensearch in a later commit. Signed-off-by: Davide Polato --- external/opensearch-java/pom.xml | 114 ------------------------------- external/opensearch/pom.xml | 17 ++--- pom.xml | 1 - 3 files changed, 5 insertions(+), 127 deletions(-) delete mode 100644 external/opensearch-java/pom.xml diff --git a/external/opensearch-java/pom.xml b/external/opensearch-java/pom.xml deleted file mode 100644 index 3a12b430b..000000000 --- a/external/opensearch-java/pom.xml +++ /dev/null @@ -1,114 +0,0 @@ - - - - - - 4.0.0 - - - org.apache.stormcrawler - stormcrawler-external - 4.0.0-SNAPSHOT - ../pom.xml - - - - 3.5.0 - 3.10.0 - true - 0.27 - 0.27 - 0.25 - 0.17 - 0.29 - 0.13 - - - stormcrawler-opensearch-java - jar - - stormcrawler-opensearch-java - - https://github.com/apache/stormcrawler/tree/master/external/opensearch-java - OpenSearch module for Apache StormCrawler using the new opensearch-java client - - - - - maven-surefire-plugin - - - default-test - test - - test - - - - - - ${opensearch.server.version} - - - - - - - - - org.opensearch.client - opensearch-java - ${opensearch.java.version} - - - - org.apache.stormcrawler - stormcrawler-core - ${project.version} - test-jar - test - - - - org.testcontainers - testcontainers - test - - - - org.testcontainers - junit-jupiter - test - - - - org.awaitility - awaitility - test - - - - org.slf4j - slf4j-simple - test - - - - diff --git a/external/opensearch/pom.xml b/external/opensearch/pom.xml index 7813de106..8f0765fb3 100644 --- a/external/opensearch/pom.xml +++ b/external/opensearch/pom.xml @@ -30,7 +30,8 @@ under the License. - 2.19.6 + 3.5.0 + 3.10.0 true 0.27 0.27 @@ -63,7 +64,7 @@ under the License. - ${opensearch.version} + ${opensearch.server.version} @@ -73,8 +74,8 @@ under the License. org.opensearch.client - opensearch-rest-high-level-client - ${opensearch.version} + opensearch-java + ${opensearch.java.version} @@ -83,14 +84,6 @@ under the License. test - - - org.opensearch.client - opensearch-rest-client-sniffer - ${opensearch.version} - - org.apache.stormcrawler stormcrawler-core diff --git a/pom.xml b/pom.xml index 2e24ae53a..6fd674320 100644 --- a/pom.xml +++ b/pom.xml @@ -747,7 +747,6 @@ under the License. external/aws external/langid external/opensearch - external/opensearch-java external/playwright external/solr external/sql From 0adea199b5d2338b01da1f5289aaa1e70c1bf33b Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Fri, 16 Oct 2026 15:58:25 +0200 Subject: [PATCH 04/14] drop the sniff and responseBufferSize settings The opensearch-java client has no sniffer and no per-request response buffer setting. The keys are removed from the module's opensearch-conf.yaml, from the one of the archetype and from the configuration reference. When they are set, OpenSearchConnection still logs a WARN, which now points to external/opensearch/README.md. The configuration reference and the Javadoc of DelegateRefresher no longer mention a second OpenSearch module. Signed-off-by: Davide Polato --- docs/src/main/asciidoc/configuration.adoc | 7 +++---- .../archetype-resources/opensearch-conf.yaml | 4 ---- external/opensearch/opensearch-conf.yaml | 13 ------------- .../stormcrawler/opensearch/DelegateRefresher.java | 4 ---- .../opensearch/OpenSearchConnection.java | 14 +++++++------- 5 files changed, 10 insertions(+), 32 deletions(-) diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 7ec62c890..f628335b7 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -355,8 +355,8 @@ Note: `indexer.md.docid` is never used raw: the matching value is passed through (the same digest used for the default URL-based ID), except for CloudSearch, whose `CloudSearchIndexerBolt` re-digests it with SHA-512 via `CloudSearchUtils.getID` to satisfy CloudSearch's ID constraints. It is only honoured by backends whose bolts call -`AbstractIndexerBolt#getDocumentID` under the hood: the OpenSearch and OpenSearch-Java -`IndexerBolt`/`DeletionBolt` pairs, and the CloudSearch `CloudSearchIndexerBolt`. The Solr and +`AbstractIndexerBolt#getDocumentID` under the hood: the OpenSearch +`IndexerBolt`/`DeletionBolt` pair, and the CloudSearch `CloudSearchIndexerBolt`. The Solr and SQL indexers build their document/row key directly from the URL and never call `getDocumentID()`, so this setting is silently ignored for those two backends. @@ -443,9 +443,8 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/opensearc | key | default value | description | opensearch.addresses | - | OpenSearch server address(es). An address without a scheme is contacted over plain http, on port 9200 if none is given; use `https://` when credentials are configured. -| opensearch.user | - | Username for Basic authentication (optional). The credentials are only sent to the addresses listed, matched on scheme, host and port, not to nodes discovered by sniffing under another address; list those nodes in the addresses or disable `opensearch..sniff`, which is on by default. A warning is logged when sniffing is enabled together with credentials, and if an address other than a loopback one uses plain http. +| opensearch.user | - | Username for Basic authentication (optional). The credentials are only sent to the addresses listed, matched on scheme, host and port. A warning is logged if an address other than a loopback one uses plain http. | opensearch.password | - | Password for authentication (optional). -| opensearch..sniff | true | Discover the other nodes of the cluster and send requests to them as well. The nodes found are contacted over https if one of the addresses uses https, over plain http otherwise. | opensearch.disable.tls.validation | false | Do not check the certificates and host names of the OpenSearch nodes. A warning is logged when this is enabled. | opensearch.concurrentRequests | 2 | Number of concurrent bulk requests. | opensearch.indexer.index.name | content | Index name for crawled documents. diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml index d1b8d4500..6152b943b 100644 --- a/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml +++ b/external/opensearch/archetype/src/main/resources/archetype-resources/opensearch-conf.yaml @@ -42,17 +42,14 @@ config: opensearch.indexer.bulkActions: 100 opensearch.indexer.flushInterval: "2s" opensearch.indexer.concurrentRequests: 1 - opensearch.indexer.sniff: true # MetricsConsumer # opensearch.metrics.addresses: "http://localhost:9200" opensearch.metrics.index.name: "metrics" - opensearch.metrics.sniff: true # Queues index opensearch.queues.addresses: "http://localhost:9200" opensearch.queues.index.name: "queues" - opensearch.queues.sniff: true # cache of known queues, see https://github.com/ben-manes/caffeine/wiki/Specification # opensearch.queues.cache.spec: "maximumSize=10000" @@ -70,7 +67,6 @@ config: opensearch.status.bulkActions: 500 opensearch.status.flushInterval: "5s" opensearch.status.concurrentRequests: 1 - opensearch.status.sniff: true # spout config # diff --git a/external/opensearch/opensearch-conf.yaml b/external/opensearch/opensearch-conf.yaml index 595e170c9..4a31a14cf 100644 --- a/external/opensearch/opensearch-conf.yaml +++ b/external/opensearch/opensearch-conf.yaml @@ -28,9 +28,6 @@ config: #opensearch.password: "PASSWORD" opensearch.concurrentRequests: 2 - # Sets the response buffer to the specified value in MB. - # opensearch.responseBufferSize: 100 - # Disable TLS validation for connection to OpenSearch, a warning is logged # when this is enabled # opensearch.disable.tls.validation: false @@ -45,22 +42,15 @@ config: opensearch.indexer.bulkActions: 100 opensearch.indexer.flushInterval: "2s" opensearch.indexer.concurrentRequests: 1 - opensearch.indexer.sniff: true - # Sets the response buffer to the specified value in MB. - # opensearch.indexer.responseBufferSize: 100 # MetricsConsumer # opensearch.metrics.addresses: "http://localhost:9200" opensearch.metrics.index.name: "metrics" - opensearch.metrics.sniff: true - # Sets the response buffer to the specified value in MB. - # opensearch.metrics.responseBufferSize: 100 # Queues index opensearch.queues.addresses: "http://localhost:9200" opensearch.queues.index.name: "queues" - opensearch.queues.sniff: true # cache of known queues, see https://github.com/ben-manes/caffeine/wiki/Specification # opensearch.queues.cache.spec: "maximumSize=10000" @@ -78,9 +68,6 @@ config: opensearch.status.bulkActions: 500 opensearch.status.flushInterval: "5s" opensearch.status.concurrentRequests: 1 - opensearch.status.sniff: true - # Sets the response buffer to the specified value in MB. - # opensearch.status.responseBufferSize: 100 # spout config # diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java index fb03a9f0f..7c49745b3 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/DelegateRefresher.java @@ -38,10 +38,6 @@ * org.apache.stormcrawler.opensearch.parse.filter.JSONResourceWrapper} to eliminate duplicated * setup/refresh/cleanup logic. * - *

This is the opensearch-java (OpenSearch Java Client 3.x / HC5) counterpart of the class with - * the same name in the {@code external/opensearch} module. It uses the typed {@link - * OpenSearchClient} instead of the deprecated {@code RestHighLevelClient}. - * * @param the base type that the delegate must extend (e.g. URLFilter or ParseFilter) */ public class DelegateRefresher { diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java index eeb473628..f822c6d70 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java @@ -276,29 +276,29 @@ private static void warnAboutPlainHttp(String boltType, List hosts) { private record ClientResources(OpenSearchClient client, OpenSearchTransport transport) {} /** - * Logs a WARN for legacy configuration keys that are no longer honoured by this module, so that - * users migrating from {@code external/opensearch} notice silently-dropped tuning. See the - * module README for the full list of differences. + * Logs a WARN for configuration keys that earlier releases of this module read and this one + * ignores, so that users upgrading notice silently-dropped tuning. See the module README for + * the full list of differences. */ private static void warnOnRemovedKeys(Map stormConf, String dottedType) { final String responseBufferKey = Constants.PARAMPREFIX + dottedType + "responseBufferSize"; if (stormConf.containsKey(responseBufferKey)) { LOG.warn( - "Configuration key '{}' is set but no longer supported by the opensearch-java module. " + "Configuration key '{}' is set but no longer supported. " + "The HC5-based async transport does not expose an equivalent per-request " + "heap-buffer override. The setting is ignored — remove it from your " - + "configuration. See external/opensearch-java/README.md for details.", + + "configuration. See external/opensearch/README.md for details.", responseBufferKey); } final String sniffKey = Constants.PARAMPREFIX + dottedType + "sniff"; if (stormConf.containsKey(sniffKey)) { LOG.warn( - "Configuration key '{}' is set but no longer supported by the opensearch-java module. " + "Configuration key '{}' is set but no longer supported. " + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so " + "automatic node discovery is not available. Keep the 'addresses' list up to " + "date manually or put a load balancer in front of the cluster. " - + "See external/opensearch-java/README.md for details.", + + "See external/opensearch/README.md for details.", sniffKey); } } From 147fe9eeb9617709bf71073658188ad20d32b8f0 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Fri, 16 Oct 2026 16:10:04 +0200 Subject: [PATCH 05/14] merge the opensearch-java README into external/opensearch The README of external/opensearch describes the client and AsyncBulkProcessor, and keeps the archetype and dashboard instructions. A new section lists what changes for users of earlier versions: the configuration keys, the behaviour differences and the types that custom code has to use. Signed-off-by: Davide Polato --- external/opensearch-java/README.md | 87 --------------------- external/opensearch/README.md | 118 +++++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 95 deletions(-) delete mode 100644 external/opensearch-java/README.md diff --git a/external/opensearch-java/README.md b/external/opensearch-java/README.md deleted file mode 100644 index 4b63e0f0c..000000000 --- a/external/opensearch-java/README.md +++ /dev/null @@ -1,87 +0,0 @@ -stormcrawler-opensearch-java -=========================== - -A collection of resources for [OpenSearch](https://opensearch.org/) built on the -[OpenSearch Java Client 3.x](https://opensearch.org/docs/latest/clients/java/) and -Apache HttpClient 5: - -* [IndexerBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java) for indexing documents crawled with StormCrawler -* [Spouts](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java) and [StatusUpdaterBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java) for persisting URL information in recursive crawls -* [MetricsConsumer](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java) -* [StatusMetricsBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/metrics/StatusMetricsBolt.java) for sending the breakdown of URLs per status as metrics and display its evolution over time. - -The bolts write to OpenSearch through `AsyncBulkProcessor`, which groups their -operations into bulk requests and: - -* sends a bulk request when the number of buffered operations reaches - `opensearch..bulkActions`, when the buffered operations reach 5 MiB of - JSON (counted before compression), or when `opensearch..flushInterval` - elapses. -* resends the items rejected with status 429 (too many requests) in the bulk - response up to 8 times, after 50, 60, 80, 150, 280, 580, 1250 and 2740 ms. - A request that OpenSearch rejects as a whole, with status 429 or any other - error, is not resent. `IndexerBolt`, `DeletionBolt` and `StatusUpdaterBolt` - fail the tuple of an item still rejected after the last retry. -* fails the whole bulk when a retry fails as a whole, for example on a - connection error: `IndexerBolt`, `DeletionBolt` and `StatusUpdaterBolt` fail - the tuples of all its items, including those stored by an earlier attempt. -* sends the requests without blocking the bolt. The bolt waits only when - `opensearch..concurrentRequests` bulks are in flight, including bulks - waiting for a retry. -* reports the outcome of a bulk to the bolt once, after its last attempt, with - the items in request order. - -Getting started ---------------------- - -Add the dependency to your crawler project: - -```xml - - org.apache.stormcrawler - stormcrawler-opensearch-java - ${stormcrawler.version} - -``` - -You will of course need to have both Storm and OpenSearch installed. For the -latter, see the [OpenSearch documentation](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/) -for Docker-based setups. - -Schemas are automatically created by the bolts on first use; you can override -them by providing your own index definitions before starting the topology. - -Configuration and dashboards ---------------------- - -For a ready-to-use crawler configuration, example Flux topologies, index -initialization scripts and OpenSearch Dashboards exports, refer to the -[`external/opensearch`](../opensearch) module: all of those resources are -compatible with this module and have not been duplicated here. - -Differences from the legacy `external/opensearch` module ---------------------- - -* `opensearch..responseBufferSize` is no longer supported. The legacy - module used the HC4-based low-level REST client and set a heap response - buffer via `HeapBufferedResponseConsumerFactory`. The HC5-based async - transport used here does not expose an equivalent per-request override, so - the key is ignored. A `WARN` is logged at startup if it is found in the - configuration; remove it when migrating. -* `opensearch..sniff` is no longer supported. The legacy module enabled - node auto-discovery by default via the low-level REST client `Sniffer`. The - OpenSearch Java Client 3.x does not ship a sniffer equivalent, so this - feature is dropped. Keep the `addresses` list up to date manually or put a - load balancer in front of the cluster. A `WARN` is logged at startup if the - key is found in the configuration; remove it when migrating. -* Date fields (`nextFetchDate` in the status index and `timestamp` in the - metrics index) are serialized as ISO-8601 strings produced by - `Instant#toString()` (for example `2026-01-01T00:00:00Z`). The legacy module - serializes the same fields through `XContentBuilder#timeField`, which emits - ISO-8601 strings via `ISODateTimeFormat.dateTime()` (for example - `2026-01-01T00:00:00.000Z`). Both representations are accepted by the - default OpenSearch `date` mapping and by the `date_optional_time` / - `strict_date_optional_time` formats used in the example mappings under - `src/test/resources/`. If you run a custom mapping that restricts the field - `format` to `epoch_millis`, update it to accept ISO-8601 (for example - `strict_date_optional_time||epoch_millis`) before writing with this module. diff --git a/external/opensearch/README.md b/external/opensearch/README.md index b3869c557..6be134b1e 100644 --- a/external/opensearch/README.md +++ b/external/opensearch/README.md @@ -1,7 +1,9 @@ stormcrawler-opensearch =========================== -A collection of resources for [OpenSearch](https://opensearch.org/): +A collection of resources for [OpenSearch](https://opensearch.org/) built on the +[OpenSearch Java Client 3.x](https://opensearch.org/docs/latest/clients/java/) and +Apache HttpClient 5: * [IndexerBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/IndexerBolt.java) for indexing documents crawled with StormCrawler * [Spouts](https://github.com/apache/stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/AggregationSpout.java) and [StatusUpdaterBolt](https://github.com/apache/stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java) for persisting URL information in recursive crawls * [MetricsConsumer](https://github.com/apache/stormcrawler/blob/master/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/metrics/MetricsConsumer.java) @@ -11,6 +13,29 @@ as well as resources for building basic real-time monitoring dashboards for the This module is ported from the Elasticsearch one. +The bolts and metrics consumers that write to OpenSearch send their operations +through `AsyncBulkProcessor`, which groups them into bulk requests. In the +configuration keys below, `` is `indexer`, `status`, `metrics` or +`queues`. `AsyncBulkProcessor`: + +* sends a bulk request when the number of buffered operations reaches + `opensearch..bulkActions`, when the buffered operations reach 5 MiB of + JSON (counted before compression), or when `opensearch..flushInterval` + elapses. +* resends the items rejected with status 429 (too many requests) in the bulk + response up to 8 times, after 50, 60, 80, 150, 280, 580, 1250 and 2740 ms. + A request that OpenSearch rejects as a whole, with status 429 or any other + error, is not resent. `IndexerBolt`, `DeletionBolt` and `StatusUpdaterBolt` + fail the tuple of an item still rejected after the last retry. +* fails the whole bulk when a retry fails as a whole, for example on a + connection error: `IndexerBolt`, `DeletionBolt` and `StatusUpdaterBolt` fail + the tuples of all its items, including those stored by an earlier attempt. +* sends the requests without blocking the bolt. The bolt waits only when + `opensearch..concurrentRequests` bulks are in flight, including bulks + waiting for a retry. +* reports the outcome of a bulk to the bolt once, after its last attempt, with + the items in request order. + Getting started --------------------- @@ -22,7 +47,7 @@ Be sure to replace with the latest released version, which you You'll be asked to enter a groupId (e.g. com.mycompany.crawler), an artefactId (e.g. stormcrawler), a version, a package name and details about the user agent to use. -This will not only create a fully formed project containing a POM with the dependency above but also a set of resources, configuration files and a topology class. Enter the directory you just created (should be the same as the artefactId you specified earlier) and follow the instructions on the README file. +This will not only create a fully formed project containing a POM with a dependency on this module but also a set of resources, configuration files and a topology class. Enter the directory you just created (should be the same as the artefactId you specified earlier) and follow the instructions on the README file. You will of course need to have both Storm and OpenSearch installed. For the latter, the [OpenSearch documentation](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/) contains resources for Docker. @@ -63,10 +88,87 @@ The crawler config YAML must be updated to use an optional argument as shown bel argument: "yyyy-MM-dd" ``` +Upgrading from versions before 4.0 +--------------------- - - - - - - +From 4.0, this module uses the OpenSearch Java Client instead of +`RestHighLevelClient` and `BulkProcessor`. + +`stormcrawler-opensearch-java`, released in 3.6.0 and 3.7.0, has the same +packages, classes and configuration keys. Its users replace that dependency with +`stormcrawler-opensearch`. For them, the 429 retries and the 5 MiB limit +described above are new, and `AsyncBulkProcessor.Builder` takes an +`OpenSearchAsyncClient`. The other changes below apply to users of +`stormcrawler-opensearch`. + +### Configuration + +* Earlier versions discovered the cluster nodes by sniffing, which was on by + default. This module sends requests only to the nodes in + `opensearch..addresses`. List all nodes there, or put a load balancer in + front of the cluster. `opensearch.sniff` and `opensearch..sniff` are + ignored. +* `opensearch.responseBufferSize` and `opensearch..responseBufferSize` are + no longer read. +* The bolts and metrics consumers that write through `AsyncBulkProcessor` log a + WARN at startup when `opensearch..sniff` or + `opensearch..responseBufferSize` is set. + Remove these keys, and their global forms, from the configuration. +* `opensearch..flushInterval` accepts a number followed by `ms`, `s` or + `m`, or a number of milliseconds without a unit. Any other value, for example + `1h`, logs a WARN and the interval falls back to 5 seconds. +* `opensearch..concurrentRequests: 0` behaves as `1`. Earlier versions + sent bulk requests synchronously with `0`. This module always sends them + asynchronously. + +### Behaviour + +* If a mapping resource (`status.mapping`, `indexer.mapping`, `metrics.mapping` + or `queues.mapping`) is missing from the classpath, the component logs a WARN + and starts without creating its index or template. Earlier versions failed at + startup. The archetype generates these files under `src/main/resources`. +* Items rejected with status 429 are resent even when other items of the same + bulk failed with another status, for example a 409 from the `create` of a URL + that is already in the status index. The `BulkProcessor` of earlier versions + retried a bulk only when every failed item had status 429. +* The 5 MiB limit counts the bytes of JSON sent to OpenSearch. Earlier versions + estimated the size from the document sources. +* `IndexerBolt` fails the tuple when building or sending a document throws any + exception. Earlier versions caught only `IOException`, and other exceptions + escaped `execute`. +* When a metadata value indexed by `IndexerBolt` is a single null, the field is + left out of the document. Earlier versions wrote it as `null`. +* `HybridSpout` sends the queries that refill its emptied queues one at a time, + on the thread that also runs the aggregation query. Earlier versions sent them + in parallel. +* The dates `nextFetchDate` in the status index and `timestamp` in the metrics + index are written with `Instant#toString()`, which leaves out the milliseconds + when they are zero (`2026-01-01T00:00:00Z`). Earlier versions always wrote + them (`2026-01-01T00:00:00.000Z`). The `date_optional_time` format of the + provided mappings accepts both. + +### Custom Java code + +Custom code that calls or extends these classes must switch to the OpenSearch +Java Client types: + +* `OpenSearchConnection.getClient(conf, type)` returns an `OpenSearchClient`. + Close it with `client._transport().close()`. +* `OpenSearchConnection.getConnection` takes an `AsyncBulkProcessor.Listener`, + and `addToProcessor` takes a `BulkOperation`. The listener receives + `org.opensearch.client.opensearch.core.BulkRequest` and `BulkResponse`. +* `AbstractSpout.client` is an `OpenSearchClient`, and `addHitToBuffer` takes + the source of a hit as a `Map`. +* `AggregationSpout` handles the search results in + `handleResponse(SearchResponse)`. The sort values passed to + `sortValuesForKey` are `FieldValue` objects. +* `IndexCreation.checkOrCreateIndex` and `checkOrCreateIndexTemplate` take an + `OpenSearchClient`. `WaitAckCache.processBulkResponse` and `processFailedBulk` + take the client's `BulkResponse` and `BulkRequest`. +* `BulkItemResponseToFailedFlag` is a record. Read `failed()`, `id()` and + `response()` instead of the public fields. `getFailure()` returns a `String`. + `status()` and the other `BulkItemResponse` methods are removed. Use + `getStatus()`, which returns an `int`, or `response()`. +* The module no longer depends on the OpenSearch server jar. Code that uses + classes such as `QueryBuilders` or `XContentBuilder` must declare that + dependency itself or use the Java client equivalents. From 61285f5f94924fb7e9e2ffd9cd48d151e677fae5 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Fri, 16 Oct 2026 16:36:49 +0200 Subject: [PATCH 06/14] port the legacy tests for credentials and WaitAckCache shutdown The legacy module tested that a running client sends the credentials to a configured address and not to another one, that credentials need both a user and a password, that host and scheme are matched without regard to case, and that WaitAckCache.shutdown() fails the tuples still waiting. The opensearch-java tests only checked how the credential scopes are built. The tests are adapted to HttpClient 5 and the opensearch-java transport. Two legacy tests are not carried over: the sniffer scheme test, since there is no sniffer, and the test for a credentials lookup without an origin, since the transport's preemptive Basic scheme looks the credentials up with the host of the request. Signed-off-by: Davide Polato --- .../OpenSearchConnectionCredentialsTest.java | 118 ++++++++++++++++++ .../opensearch/WaitAckCacheTest.java | 16 +++ 2 files changed, 134 insertions(+) diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java index bb484e843..ba8e176e5 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionCredentialsTest.java @@ -23,12 +23,30 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.IntFunction; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.core5.http.HttpHost; import org.junit.jupiter.api.Test; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.generic.Requests; +import org.opensearch.client.opensearch.generic.Response; +import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; class OpenSearchConnectionCredentialsTest { @@ -98,6 +116,106 @@ private static AuthScope request(String scheme, String host, int port) { return new AuthScope(new HttpHost(scheme, host, port), null, "Basic"); } + @Test + void credentialsAreGivenToTheConfiguredAddresses() { + final BasicCredentialsProvider provider = + providerFor( + List.of( + new HttpHost("https", "opensearch1.example.org", 9200), + new HttpHost("https", "OpenSearch2.example.org", 9201))); + assertNotNull( + provider.getCredentials(request("https", "opensearch1.example.org", 9200), null)); + assertNotNull( + provider.getCredentials(request("HTTPS", "opensearch2.example.org", 9201), null)); + } + + /** + * Sends a request with the client built for the port of a local server and returns the + * Authorization header the server received, if any. + */ + private static String authorizationSentTo(IntFunction clientForPort) + throws Exception { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) { + final CompletableFuture authorization = + CompletableFuture.supplyAsync( + () -> { + try (Socket socket = server.accept()) { + final BufferedReader in = + new BufferedReader( + new InputStreamReader( + socket.getInputStream(), + StandardCharsets.US_ASCII)); + String header = null; + for (String line = in.readLine(); + line != null && !line.isEmpty(); + line = in.readLine()) { + if (line.regionMatches(true, 0, "Authorization:", 0, 14)) { + header = line.substring(14).trim(); + } + } + socket.getOutputStream() + .write( + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .getBytes(StandardCharsets.US_ASCII)); + return header; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + final OpenSearchClient client = clientForPort.apply(server.getLocalPort()); + try (Response response = + client.generic() + .execute(Requests.builder().endpoint("/").method("GET").build())) { + return authorization.get(10, TimeUnit.SECONDS); + } finally { + client._transport().close(); + } + } + } + + private static OpenSearchClient connectionClient(int port, String user, String password) { + final Map conf = new HashMap<>(); + conf.put("opensearch.test.addresses", "http://127.0.0.1:" + port); + conf.put("opensearch.test.user", user); + conf.put("opensearch.test.password", password); + return OpenSearchConnection.getClient(conf, "test"); + } + + @Test + void clientSendsTheCredentialsToAConfiguredAddress() throws Exception { + assertEquals( + "Basic " + + Base64.getEncoder() + .encodeToString("crawler:s3cret".getBytes(StandardCharsets.UTF_8)), + authorizationSentTo(port -> connectionClient(port, "crawler", "s3cret"))); + } + + @Test + void clientSendsNoCredentialsToAnotherAddress() throws Exception { + assertNull( + authorizationSentTo( + port -> + new OpenSearchClient( + ApacheHttpClient5TransportBuilder.builder( + new HttpHost("http", "127.0.0.1", port)) + .setHttpClientConfigCallback( + b -> + b.setDefaultCredentialsProvider( + providerFor( + List.of( + new HttpHost( + "https", + "127.0.0.1", + port))))) + .build()))); + } + + @Test + void credentialsNeedUserAndPassword() throws Exception { + assertNull(authorizationSentTo(port -> connectionClient(port, "crawler", " "))); + assertNull(authorizationSentTo(port -> connectionClient(port, null, "s3cret"))); + } + @Test void plainHttpToARemoteHostIsReported() { final HttpHost remote = new HttpHost("http", "opensearch1.example.org", 9200); diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java index 3bd2dc6d4..0b7954a62 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/WaitAckCacheTest.java @@ -333,4 +333,20 @@ void processBulkResponse_multipleDocIds() { assertEquals(1, failed.size()); assertSame(t2, failed.get(0)); } + + @Test + void shutdown_failsAllRemainingTuples() { + Tuple t1 = mockTuple("http://example.com/1"); + Tuple t2 = mockTuple("http://example.com/2"); + cache.addTuple("doc1", t1); + cache.addTuple("doc2", t2); + + cache.shutdown(); + + assertEquals(2, evicted.size()); + assertTrue(evicted.contains(t1)); + assertTrue(evicted.contains(t2)); + assertFalse(cache.contains("doc1")); + assertFalse(cache.contains("doc2")); + } } From 3ef3a72b0758bc8ef7dc0c4fe4c8e059730b6591 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Fri, 16 Oct 2026 16:44:24 +0200 Subject: [PATCH 07/14] keep the connection pool limits when TLS validation is disabled With opensearch.disable.tls.validation set, OpenSearchConnection replaces the connection manager of the transport to install a trust-all TLS strategy. The replacement had no pool limits, so HttpClient 5 allowed 5 connections per node and 25 in total, instead of the 10 and 30 that the transport sets. The replacement now uses the transport's limits. The new test sends 12 requests to a local server that never answers and checks that 10 connections are open. Signed-off-by: Davide Polato --- .../opensearch/OpenSearchConnection.java | 6 +- .../opensearch/OpenSearchConnectionTest.java | 62 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java index f822c6d70..d8ca3011d 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java @@ -18,6 +18,8 @@ package org.apache.stormcrawler.opensearch; import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_CONNECT_TIMEOUT_MILLIS; +import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_MAX_CONN_PER_ROUTE; +import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_MAX_CONN_TOTAL; import static org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder.DEFAULT_RESPONSE_TIMEOUT_MILLIS; import java.io.IOException; @@ -435,7 +437,7 @@ private static ClientResources buildClientResources( new HttpHost(proxyscheme, proxyhost, proxyport)); } // Custom connection manager overrides the builder's internal one, - // so timeouts and TlsDetailsFactory must be replicated here + // so pool limits, timeouts and TlsDetailsFactory must be replicated here if (disableTlsValidation) { try { final SSLContext sslContext = @@ -444,6 +446,8 @@ private static ClientResources buildClientResources( .build(); httpClientBuilder.setConnectionManager( PoolingAsyncClientConnectionManagerBuilder.create() + .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE) + .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL) .setTlsStrategy( ClientTlsStrategyBuilder.create() .setSslContext(sslContext) diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java index 8c5789a1c..19b39ce6d 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/OpenSearchConnectionTest.java @@ -17,9 +17,23 @@ package org.apache.stormcrawler.opensearch; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; +import java.io.IOException; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.junit.jupiter.api.Test; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.generic.Requests; class OpenSearchConnectionTest { @@ -72,4 +86,52 @@ void invalidWithSuffixReturnsDefault() { void whitespaceIsTrimmed() { assertEquals(5000, OpenSearchConnection.parseTimeValueToMillis(" 5s ", 0)); } + + @Test + void disabledTlsValidationKeepsTenConnectionsPerNode() throws Exception { + try (ServerSocket server = new ServerSocket(0, 50, InetAddress.getLoopbackAddress())) { + // accepts the connections and never answers, so each request keeps its connection + final List accepted = new CopyOnWriteArrayList<>(); + final Thread acceptor = + new Thread( + () -> { + try { + while (true) { + accepted.add(server.accept()); + } + } catch (IOException e) { + // server closed + } + }); + acceptor.setDaemon(true); + acceptor.start(); + + final Map conf = new HashMap<>(); + conf.put("opensearch.test.addresses", "http://127.0.0.1:" + server.getLocalPort()); + conf.put("opensearch.disable.tls.validation", true); + final OpenSearchClient client = OpenSearchConnection.getClient(conf, "test"); + final ExecutorService requests = Executors.newFixedThreadPool(12); + try { + for (int i = 0; i < 12; i++) { + requests.submit( + () -> + client.generic() + .execute( + Requests.builder() + .endpoint("/") + .method("GET") + .build())); + } + await().during(Duration.ofMillis(300)) + .atMost(Duration.ofSeconds(5)) + .untilAsserted(() -> assertEquals(10, accepted.size())); + } finally { + for (Socket socket : accepted) { + socket.close(); + } + requests.shutdownNow(); + client._transport().close(); + } + } + } } From 8ff2ced0119f85516b87903f65daf1a778cfefa8 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Fri, 16 Oct 2026 17:48:24 +0200 Subject: [PATCH 08/14] upgrade server Signed-off-by: Davide Polato --- external/opensearch/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/opensearch/pom.xml b/external/opensearch/pom.xml index 8f0765fb3..14067b739 100644 --- a/external/opensearch/pom.xml +++ b/external/opensearch/pom.xml @@ -30,7 +30,7 @@ under the License. - 3.5.0 + 3.9.0 3.10.0 true 0.27 From 3457e7059c710c3347d6c946f0ec74611fd3cb85 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Mon, 19 Oct 2026 10:48:19 +0200 Subject: [PATCH 09/14] turn off the bulk count limit for a negative bulkActions The legacy BulkProcessor treats bulkActions -1 as no limit on the number of operations. AsyncBulkProcessor used the value as the capacity of its buffer, so -1 made the bolt fail at startup. A negative value now turns off the count limit, while the size limit and the flush interval still apply. 0 keeps sending each operation in its own bulk, as in both modules. Signed-off-by: Davide Polato --- external/opensearch/README.md | 3 +- .../opensearch/AsyncBulkProcessor.java | 11 +++-- .../opensearch/AsyncBulkProcessorTest.java | 44 +++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/external/opensearch/README.md b/external/opensearch/README.md index 6be134b1e..5ff9942e5 100644 --- a/external/opensearch/README.md +++ b/external/opensearch/README.md @@ -21,7 +21,8 @@ configuration keys below, `` is `indexer`, `status`, `metrics` or * sends a bulk request when the number of buffered operations reaches `opensearch..bulkActions`, when the buffered operations reach 5 MiB of JSON (counted before compression), or when `opensearch..flushInterval` - elapses. + elapses. A negative `bulkActions` turns off the limit on the number of + operations. * resends the items rejected with status 429 (too many requests) in the bulk response up to 8 times, after 50, 60, 80, 150, 280, 580, 1250 and 2740 ms. A request that OpenSearch rejects as a whole, with status 429 or any other diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java index 85866fa09..a51fb1564 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java @@ -104,7 +104,7 @@ private AsyncBulkProcessor(Builder builder) { this.bulkActions = builder.bulkActions; this.concurrentRequests = builder.concurrentRequests; this.concurrencyPermits = new Semaphore(this.concurrentRequests); - this.buffer = new ArrayList<>(bulkActions); + this.buffer = new ArrayList<>(); this.scheduler = daemonScheduler("AsyncBulkProcessor-flush"); this.retryScheduler = @@ -143,7 +143,8 @@ public void add(BulkOperation operation) { try { buffer.add(operation); bufferedBytes += size; - if (buffer.size() >= bulkActions || bufferedBytes >= BULK_SIZE_BYTES) { + if ((bulkActions >= 0 && buffer.size() >= bulkActions) + || bufferedBytes >= BULK_SIZE_BYTES) { toFlush = swapBuffer(); } } finally { @@ -176,7 +177,7 @@ private void flushIfNeeded() { */ private List swapBuffer() { List old = buffer; - buffer = new ArrayList<>(bulkActions); + buffer = new ArrayList<>(); bufferedBytes = 0; return old; } @@ -400,6 +401,10 @@ public Builder(OpenSearchAsyncClient client, Listener listener) { this.listener = listener; } + /** + * Sets the number of buffered operations that triggers a bulk request. A negative value + * turns off this limit; the size limit and the flush interval still apply. + */ public Builder setBulkActions(int bulkActions) { this.bulkActions = bulkActions; return this; diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java index 0c91e5a88..f78d1f935 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessorTest.java @@ -25,7 +25,9 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -112,6 +114,48 @@ public void afterBulk( processor.awaitClose(5, TimeUnit.SECONDS); } + /** Adds three operations to a processor with the given bulkActions, then closes it. */ + private static List bulkSizes(int bulkActions) throws Exception { + List sizes = new CopyOnWriteArrayList<>(); + AsyncBulkProcessor.Listener listener = + new AsyncBulkProcessor.Listener() { + @Override + public void beforeBulk(long executionId, BulkRequest request) {} + + @Override + public void afterBulk( + long executionId, BulkRequest request, BulkResponse response) { + sizes.add(request.operations().size()); + } + + @Override + public void afterBulk( + long executionId, BulkRequest request, Throwable failure) {} + }; + AsyncBulkProcessor processor = + new AsyncBulkProcessor.Builder(mockClient(), listener) + .setBulkActions(bulkActions) + .setFlushIntervalMillis(60_000) + .build(); + processor.add(dummyOp()); + processor.add(dummyOp()); + processor.add(dummyOp()); + assertTrue(processor.awaitClose(5, TimeUnit.SECONDS)); + return sizes; + } + + @Test + @Timeout(10) + void negativeBulkActionsTurnsOffTheCountLimit() throws Exception { + assertEquals(List.of(3), bulkSizes(-1)); + } + + @Test + @Timeout(10) + void zeroBulkActionsSendsEachOperationInItsOwnBulk() throws Exception { + assertEquals(List.of(1, 1, 1), bulkSizes(0)); + } + /** Verify that the timer-based flush fires even when bulkActions threshold is not reached. */ @Test @Timeout(10) From 41dc5ef44306f20588f384516b32c2a23dafea9e Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Mon, 19 Oct 2026 10:48:34 +0200 Subject: [PATCH 10/14] warn about the global sniff and responseBufferSize keys OpenSearchConnection logged a WARN only for the per-type keys, so a global opensearch.sniff or opensearch.responseBufferSize was ignored without a warning. The WARN now covers both forms and names the key that is set. Signed-off-by: Davide Polato --- external/opensearch/README.md | 5 +-- .../opensearch/OpenSearchConnection.java | 43 ++++++++++--------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/external/opensearch/README.md b/external/opensearch/README.md index 5ff9942e5..f0078c772 100644 --- a/external/opensearch/README.md +++ b/external/opensearch/README.md @@ -112,9 +112,8 @@ described above are new, and `AsyncBulkProcessor.Builder` takes an * `opensearch.responseBufferSize` and `opensearch..responseBufferSize` are no longer read. * The bolts and metrics consumers that write through `AsyncBulkProcessor` log a - WARN at startup when `opensearch..sniff` or - `opensearch..responseBufferSize` is set. - Remove these keys, and their global forms, from the configuration. + WARN at startup when `opensearch.sniff`, `opensearch.responseBufferSize` or + their per-type forms are set. Remove these keys from the configuration. * `opensearch..flushInterval` accepts a number followed by `ms`, `s` or `m`, or a number of milliseconds without a unit. Any other value, for example `1h`, logs a WARN and the interval falls back to 5 seconds. diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java index d8ca3011d..9378c2e95 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java @@ -278,30 +278,31 @@ private static void warnAboutPlainHttp(String boltType, List hosts) { private record ClientResources(OpenSearchClient client, OpenSearchTransport transport) {} /** - * Logs a WARN for configuration keys that earlier releases of this module read and this one - * ignores, so that users upgrading notice silently-dropped tuning. See the module README for - * the full list of differences. + * Logs a WARN for configuration keys, per type or global, that earlier releases of this module + * read and this one ignores. See the module README for the full list of differences. */ private static void warnOnRemovedKeys(Map stormConf, String dottedType) { - final String responseBufferKey = Constants.PARAMPREFIX + dottedType + "responseBufferSize"; - if (stormConf.containsKey(responseBufferKey)) { - LOG.warn( - "Configuration key '{}' is set but no longer supported. " - + "The HC5-based async transport does not expose an equivalent per-request " - + "heap-buffer override. The setting is ignored — remove it from your " - + "configuration. See external/opensearch/README.md for details.", - responseBufferKey); - } + for (String prefix : List.of(Constants.PARAMPREFIX + dottedType, Constants.PARAMPREFIX)) { + final String responseBufferKey = prefix + "responseBufferSize"; + if (stormConf.containsKey(responseBufferKey)) { + LOG.warn( + "Configuration key '{}' is set but no longer supported. " + + "The HC5-based async transport does not expose an equivalent per-request " + + "heap-buffer override. The setting is ignored — remove it from your " + + "configuration. See external/opensearch/README.md for details.", + responseBufferKey); + } - final String sniffKey = Constants.PARAMPREFIX + dottedType + "sniff"; - if (stormConf.containsKey(sniffKey)) { - LOG.warn( - "Configuration key '{}' is set but no longer supported. " - + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so " - + "automatic node discovery is not available. Keep the 'addresses' list up to " - + "date manually or put a load balancer in front of the cluster. " - + "See external/opensearch/README.md for details.", - sniffKey); + final String sniffKey = prefix + "sniff"; + if (stormConf.containsKey(sniffKey)) { + LOG.warn( + "Configuration key '{}' is set but no longer supported. " + + "The OpenSearch Java Client 3.x does not ship a Sniffer equivalent, so " + + "automatic node discovery is not available. Keep the 'addresses' list up to " + + "date manually or put a load balancer in front of the cluster. " + + "See external/opensearch/README.md for details.", + sniffKey); + } } } From 265f122afecfd96dc9dc80908d6433f14628ea5c Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Mon, 19 Oct 2026 10:48:37 +0200 Subject: [PATCH 11/14] run OpenSearch 3.9.0 in the archetype's docker-compose.yml The module tests run against OpenSearch 3.9.0, while the archetype's compose file still started 2.19.4 for OpenSearch and Dashboards. AbstractOpenSearchTest now also falls back to 3.9.0 when the opensearch-version property is not set, as in a run from an IDE. Signed-off-by: Davide Polato --- .../src/main/resources/archetype-resources/docker-compose.yml | 4 ++-- .../stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/docker-compose.yml b/external/opensearch/archetype/src/main/resources/archetype-resources/docker-compose.yml index dbda07d1f..a15909f01 100644 --- a/external/opensearch/archetype/src/main/resources/archetype-resources/docker-compose.yml +++ b/external/opensearch/archetype/src/main/resources/archetype-resources/docker-compose.yml @@ -57,7 +57,7 @@ services: - "127.0.0.1:8000:8000" opensearch-sc: - image: opensearchproject/opensearch:2.19.4 + image: opensearchproject/opensearch:3.9.0 container_name: opensearch-sc environment: - cluster.name=opensearch-sc-cluster @@ -80,7 +80,7 @@ services: - "127.0.0.1:9200:9200" # REST API opensearch-dashboard: - image: opensearchproject/opensearch-dashboards:2.19.4 + image: opensearchproject/opensearch-dashboards:3.9.0 container_name: dashboard ports: - "127.0.0.1:5601:5601" diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java index 44150ea67..71dc13e4a 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/bolt/AbstractOpenSearchTest.java @@ -27,7 +27,7 @@ public abstract class AbstractOpenSearchTest { private static final String OPENSEARCH_VERSION = - System.getProperty("opensearch-version", "2.19.5"); + System.getProperty("opensearch-version", "3.9.0"); public static final String PASSWORD = "This1sAPassw0rd"; From 3cb58500a455f523c4f82fead730c50d86cb71db Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Tue, 20 Oct 2026 16:56:29 +0200 Subject: [PATCH 12/14] fail at startup when a mapping resource is missing IndexCreation logged a WARN when the .mapping resource was not on the classpath, and the component started without its index or template. It now looks the resource up with Resources.getResource before the try, which throws IllegalArgumentException naming the resource, so the bolt, spout or metrics consumer fails in prepare() or open(). The README no longer lists the WARN among the changes in behaviour. Signed-off-by: Davide Polato --- external/opensearch/README.md | 4 - .../opensearch/IndexCreation.java | 23 ++++-- .../opensearch/IndexCreationTest.java | 75 +++++++++++++++++++ 3 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/IndexCreationTest.java diff --git a/external/opensearch/README.md b/external/opensearch/README.md index f0078c772..5fe60d034 100644 --- a/external/opensearch/README.md +++ b/external/opensearch/README.md @@ -123,10 +123,6 @@ described above are new, and `AsyncBulkProcessor.Builder` takes an ### Behaviour -* If a mapping resource (`status.mapping`, `indexer.mapping`, `metrics.mapping` - or `queues.mapping`) is missing from the classpath, the component logs a WARN - and starts without creating its index or template. Earlier versions failed at - startup. The archetype generates these files under `src/main/resources`. * Items rejected with status 429 are resent even when other items of the same bulk failed with another status, for example a 409 from the `create` of a URL that is already in the status index. The `BulkProcessor` of earlier versions diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java index f76172057..b41a6c384 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java @@ -29,6 +29,12 @@ public class IndexCreation { + /** + * Creates the index with the {@code .mapping} resource if it does not exist. + * + * @throws IllegalArgumentException if the index does not exist and the resource is not on the + * classpath + */ public static synchronized void checkOrCreateIndex( OpenSearchClient client, String indexName, String boltType, Logger log) throws IOException { @@ -43,6 +49,13 @@ public static synchronized void checkOrCreateIndex( } } + /** + * Creates the template {@code -template} with the {@code .mapping} resource + * if it does not exist. + * + * @throws IllegalArgumentException if the template does not exist and the resource is not on + * the classpath + */ public static synchronized void checkOrCreateIndexTemplate( OpenSearchClient client, String boltType, Logger log) throws IOException { final String templateName = boltType + "-template"; @@ -63,10 +76,9 @@ public static synchronized void checkOrCreateIndexTemplate( private static boolean createTemplate( OpenSearchClient client, String templateName, String resourceName, Logger log) { - try { - final URL mapping = - Thread.currentThread().getContextClassLoader().getResource(resourceName); + final URL mapping = Resources.getResource(resourceName); + try { final String jsonIndexConfiguration = Resources.toString(mapping, StandardCharsets.UTF_8); @@ -90,10 +102,9 @@ private static boolean createTemplate( private static boolean createIndex( OpenSearchClient client, String indexName, String resourceName, Logger log) { - try { - final URL mapping = - Thread.currentThread().getContextClassLoader().getResource(resourceName); + final URL mapping = Resources.getResource(resourceName); + try { final String jsonIndexConfiguration = Resources.toString(mapping, StandardCharsets.UTF_8); diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/IndexCreationTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/IndexCreationTest.java new file mode 100644 index 000000000..cf527bf03 --- /dev/null +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/IndexCreationTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.opensearch; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.function.Function; +import org.junit.jupiter.api.Test; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.indices.ExistsTemplateRequest; +import org.opensearch.client.opensearch.indices.OpenSearchIndicesClient; +import org.opensearch.client.transport.endpoints.BooleanResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class IndexCreationTest { + + private static final Logger LOG = LoggerFactory.getLogger(IndexCreationTest.class); + + /** A client on which neither the index nor the template exists. */ + @SuppressWarnings("unchecked") + private static OpenSearchClient emptyCluster() throws IOException { + OpenSearchClient client = mock(OpenSearchClient.class); + OpenSearchIndicesClient indices = mock(OpenSearchIndicesClient.class); + when(client.indices()).thenReturn(indices); + when(indices.exists(any(Function.class))).thenReturn(new BooleanResponse(false)); + when(indices.existsTemplate(any(ExistsTemplateRequest.class))) + .thenReturn(new BooleanResponse(false)); + return client; + } + + @Test + void indexCreationFailsWhenTheMappingIsNotOnTheClasspath() throws IOException { + OpenSearchClient client = emptyCluster(); + + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> IndexCreation.checkOrCreateIndex(client, "idx", "missing", LOG)); + + assertTrue(e.getMessage().contains("missing.mapping")); + } + + @Test + void templateCreationFailsWhenTheMappingIsNotOnTheClasspath() throws IOException { + OpenSearchClient client = emptyCluster(); + + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> IndexCreation.checkOrCreateIndexTemplate(client, "missing", LOG)); + + assertTrue(e.getMessage().contains("missing.mapping")); + } +} From 8c594f2c6c54176d7d956bcce6327b7e2a67a26b Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Wed, 21 Oct 2026 17:19:21 +0200 Subject: [PATCH 13/14] minor enhancement: log why an index or template was not created createIndex and createTemplate returned false when OpenSearch answered the creation with an error status, and checkOrCreateIndex only logged that the index was not created. They now log a WARN with the status and the response body, as the legacy module did with the exception of the high-level client. The comments that said they only log exceptions are removed. IndexCreationTest checks both paths with a mocked 400. --- .../opensearch/IndexCreation.java | 23 +++++++-- .../opensearch/IndexCreationTest.java | 47 +++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java index b41a6c384..2717231cb 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/IndexCreation.java @@ -22,6 +22,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.generic.Body; import org.opensearch.client.opensearch.generic.Requests; import org.opensearch.client.opensearch.generic.Response; import org.opensearch.client.opensearch.indices.ExistsTemplateRequest; @@ -41,7 +42,6 @@ public static synchronized void checkOrCreateIndex( final boolean indexExists = client.indices().exists(req -> req.index(indexName)).value(); log.info("Index '{}' exists? {}", indexName, indexExists); // there's a possible check-then-update race condition - // createIndex intentionally catches and logs exceptions from OpenSearch if (!indexExists) { boolean created = IndexCreation.createIndex(client, indexName, boltType + ".mapping", log); @@ -65,7 +65,6 @@ public static synchronized void checkOrCreateIndexTemplate( .value(); log.info("Template '{}' exists? {}", templateName, templateExists); // there's a possible check-then-update race condition - // createTemplate intentionally catches and logs exceptions from OpenSearch if (!templateExists) { boolean created = IndexCreation.createTemplate(client, templateName, boltType + ".mapping", log); @@ -91,7 +90,15 @@ private static boolean createTemplate( .json(jsonIndexConfiguration) .build())) { int statusCode = response.getStatus(); - return statusCode == 200 || statusCode == 201; + if (statusCode == 200 || statusCode == 201) { + return true; + } + log.warn( + "template '{}' not created: {} {}", + templateName, + statusCode, + response.getBody().map(Body::bodyAsString).orElse("")); + return false; } } catch (Exception e) { log.warn("template '{}' not created", templateName, e); @@ -117,7 +124,15 @@ private static boolean createIndex( .json(jsonIndexConfiguration) .build())) { int statusCode = response.getStatus(); - return statusCode == 200 || statusCode == 201; + if (statusCode == 200 || statusCode == 201) { + return true; + } + log.warn( + "index '{}' not created: {} {}", + indexName, + statusCode, + response.getBody().map(Body::bodyAsString).orElse("")); + return false; } } catch (Exception e) { log.warn("index '{}' not created", indexName, e); diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/IndexCreationTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/IndexCreationTest.java index cf527bf03..49beae7d4 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/IndexCreationTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/IndexCreationTest.java @@ -20,13 +20,22 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; import java.util.function.Function; import org.junit.jupiter.api.Test; import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.generic.Body; +import org.opensearch.client.opensearch.generic.OpenSearchGenericClient; +import org.opensearch.client.opensearch.generic.Request; +import org.opensearch.client.opensearch.generic.Response; import org.opensearch.client.opensearch.indices.ExistsTemplateRequest; import org.opensearch.client.opensearch.indices.OpenSearchIndicesClient; import org.opensearch.client.transport.endpoints.BooleanResponse; @@ -49,6 +58,44 @@ private static OpenSearchClient emptyCluster() throws IOException { return client; } + private static final String REJECTION = + "{\"error\":{\"type\":\"mapper_parsing_exception\"},\"status\":400}"; + + /** A client on which neither the index nor the template exists and that rejects both. */ + private static OpenSearchClient rejectingCluster() throws IOException { + OpenSearchClient client = emptyCluster(); + Response response = mock(Response.class); + when(response.getStatus()).thenReturn(400); + when(response.getBody()) + .thenReturn( + Optional.of( + Body.from( + REJECTION.getBytes(StandardCharsets.UTF_8), + "application/json"))); + OpenSearchGenericClient generic = mock(OpenSearchGenericClient.class); + when(generic.execute(any(Request.class))).thenReturn(response); + when(client.generic()).thenReturn(generic); + return client; + } + + @Test + void rejectedIndexIsLoggedWithStatusAndBody() throws IOException { + Logger log = mock(Logger.class); + + IndexCreation.checkOrCreateIndex(rejectingCluster(), "idx", "status", log); + + verify(log).warn(anyString(), eq("idx"), eq(400), eq(REJECTION)); + } + + @Test + void rejectedTemplateIsLoggedWithStatusAndBody() throws IOException { + Logger log = mock(Logger.class); + + IndexCreation.checkOrCreateIndexTemplate(rejectingCluster(), "status", log); + + verify(log).warn(anyString(), eq("status-template"), eq(400), eq(REJECTION)); + } + @Test void indexCreationFailsWhenTheMappingIsNotOnTheClasspath() throws IOException { OpenSearchClient client = emptyCluster(); From cddff761262fa6d2da59abf72a93fa8dc43c0c0d Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Wed, 21 Oct 2026 17:19:34 +0200 Subject: [PATCH 14/14] drop or shorten comments --- external/opensearch/README.md | 7 +++--- .../opensearch/AsyncBulkProcessor.java | 23 ++++++++---------- .../BulkItemResponseToFailedFlag.java | 1 - .../opensearch/OpenSearchConnection.java | 8 +------ .../stormcrawler/opensearch/WaitAckCache.java | 5 ---- .../persistence/StatusUpdaterBolt.java | 1 - .../persistence/AbstractSpoutTest.java | 24 +++++-------------- 7 files changed, 21 insertions(+), 48 deletions(-) diff --git a/external/opensearch/README.md b/external/opensearch/README.md index 5fe60d034..bc5a3d58b 100644 --- a/external/opensearch/README.md +++ b/external/opensearch/README.md @@ -114,9 +114,10 @@ described above are new, and `AsyncBulkProcessor.Builder` takes an * The bolts and metrics consumers that write through `AsyncBulkProcessor` log a WARN at startup when `opensearch.sniff`, `opensearch.responseBufferSize` or their per-type forms are set. Remove these keys from the configuration. -* `opensearch..flushInterval` accepts a number followed by `ms`, `s` or - `m`, or a number of milliseconds without a unit. Any other value, for example - `1h`, logs a WARN and the interval falls back to 5 seconds. +* `opensearch..flushInterval` takes a positive integer followed by `ms`, + `s` or `m`, or a positive integer of milliseconds without a unit. Zero or a + negative value makes the component fail at startup. Any other value, for + example `1h` or `1.5s`, logs a WARN and the interval falls back to 5 seconds. * `opensearch..concurrentRequests: 0` behaves as `1`. Earlier versions sent bulk requests synchronously with `0`. This module always sends them asynchronously. diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java index a51fb1564..92bb8b6d6 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/AsyncBulkProcessor.java @@ -44,16 +44,13 @@ import org.slf4j.LoggerFactory; /** - * Replacement for the legacy {@code org.opensearch.action.bulk.BulkProcessor} that works with the - * new opensearch-java client. Accumulates {@link BulkOperation} instances and flushes them to - * OpenSearch when the configured number of actions is reached, when the buffered operations reach 5 - * MiB, or when a periodic timer fires. + * Accumulates {@link BulkOperation} instances and flushes them to OpenSearch when the configured + * number of actions is reached, when the buffered operations reach 5 MiB, or when a periodic timer + * fires. * - *

Bulk requests are sent with the {@link OpenSearchAsyncClient}, so the calling thread never - * waits for the HTTP round trip. Concurrency is controlled via a {@link Semaphore}: each in-flight - * bulk request acquires a permit, which is only released once the listener has processed the - * outcome. This provides natural back-pressure towards the Storm topology when the cluster slows - * down. + *

Bulk requests are sent with the {@link OpenSearchAsyncClient}. Each in-flight bulk request + * holds a {@link Semaphore} permit, released once the listener has processed its outcome, so the + * calling thread waits for a permit when the cluster slows down. * *

Items rejected with status 429 are sent again up to 8 times, while the bulk keeps its permit. * The listener's {@code afterBulk} is called once per bulk, with the items of all attempts in @@ -68,7 +65,7 @@ public final class AsyncBulkProcessor implements AutoCloseable { /** Flush threshold, in bytes of the serialized operations before compression. */ static final long BULK_SIZE_BYTES = 5L * 1024 * 1024; - /** Listener interface equivalent to the legacy {@code BulkProcessor.Listener}. */ + /** Callbacks for the first send of a bulk and for its outcome, retries included. */ public interface Listener { void beforeBulk(long executionId, BulkRequest request); @@ -135,7 +132,7 @@ private static ScheduledExecutorService daemonScheduler(String name) { */ public void add(BulkOperation operation) { if (closed.get()) { - throw new IllegalStateException("BulkProcessor is closed"); + throw new IllegalStateException("AsyncBulkProcessor is closed"); } final long size = serializedSize(mapper, operation); List toFlush = null; @@ -290,8 +287,8 @@ private void finish(Bulk bulk, BulkResponse response, Throwable failure) { } /** - * Drains pending operations and waits for all in-flight bulk requests to complete, up to the - * given timeout. Equivalent to the legacy {@code BulkProcessor.awaitClose()}. + * Flushes the pending operations, then waits up to the given timeout for the in-flight bulk + * requests, retries included. The flush itself can wait for a concurrency permit. * * @return {@code true} if all operations completed within the timeout */ diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java index c32f162f2..316077a65 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/BulkItemResponseToFailedFlag.java @@ -63,7 +63,6 @@ public String getFailure() { return error.reason() != null ? error.reason() : error.type(); } - // opensearch-java: status() returns int HTTP code, not RestStatus enum /** Returns the HTTP status code of this response item. */ public int getStatus() { return response.status(); diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java index 9378c2e95..ec50defc0 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/OpenSearchConnection.java @@ -186,12 +186,11 @@ public void close() { LOG.debug("Start closing the OpenSearch connection"); - // First, close the BulkProcessor ensuring pending actions are flushed try { boolean success = processor.awaitClose(60, TimeUnit.SECONDS); if (!success) { throw new RuntimeException( - "Failed to flush pending actions when closing BulkProcessor"); + "Failed to flush pending actions when closing AsyncBulkProcessor"); } } catch (InterruptedException e) { throw new RuntimeException(e); @@ -343,7 +342,6 @@ private static ClientResources buildClientResources( if (uri.getScheme() != null) { scheme = uri.getScheme(); } - // HC5: constructor is (scheme, hostname, port) — not (hostname, port, scheme) hosts.add(new HttpHost(scheme, uri.getHost(), port)); } @@ -376,7 +374,6 @@ private static ClientResources buildClientResources( final boolean needsUser = StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password); final boolean needsProxy = StringUtils.isNotBlank(proxyhost) && proxyport != -1; - // Defaults from ApacheHttpClient5TransportBuilder (same as the former RestClientBuilder) final int connectTimeout = ConfUtils.getInt( stormConf, @@ -421,7 +418,6 @@ private static ClientResources buildClientResources( if (needsUser || needsProxy || disableTlsValidation) { builder.setHttpClientConfigCallback( httpClientBuilder -> { - // hc.client5 auth: password is char[], AuthScope(host, port) if (needsUser) { final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); @@ -432,7 +428,6 @@ private static ClientResources buildClientResources( } httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); } - // hc.client5 proxy: HttpHost(scheme, host, port) if (needsProxy) { httpClientBuilder.setProxy( new HttpHost(proxyscheme, proxyhost, proxyport)); @@ -482,7 +477,6 @@ private static ClientResources buildClientResources( }); } - // Compression: first-class builder method, not a request interceptor builder.setCompressionEnabled(compression); final OpenSearchTransport transport = builder.build(); diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java index 0e8574fbb..964635f97 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/WaitAckCache.java @@ -34,7 +34,6 @@ import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.metrics.ScopedCounter; import org.jetbrains.annotations.Nullable; -// opensearch-java: uses typed BulkRequest/BulkResponse, not legacy REST equivalents import org.opensearch.client.opensearch.core.BulkRequest; import org.opensearch.client.opensearch.core.BulkResponse; import org.slf4j.Logger; @@ -178,7 +177,6 @@ public void processBulkResponse( @Nullable ScopedCounter conflictCounter, TupleAction action) { - // opensearch-java: items() returns List; status() returns int var idsToBulkItems = response.items().stream() .map( @@ -186,7 +184,6 @@ public void processBulkResponse( var error = bir.error(); boolean failed = false; if (error != null) { - // opensearch-java: int status code, not RestStatus enum if (bir.status() == 409) { if (conflictCounter != null) { conflictCounter.scope("doc_conflicts").incrBy(1); @@ -277,8 +274,6 @@ public void processFailedBulk( log.error("Exception with bulk {} - failing the whole lot ", executionId, failure); - // opensearch-java: operations() + getBulkOperationId replaces - // legacy requests() + DocWriteRequest::id final var failedIds = request.operations().stream() .map(OpenSearchConnection::getBulkOperationId) diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java index 2e0b92eb8..bdc0457e7 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/persistence/StatusUpdaterBolt.java @@ -148,7 +148,6 @@ public void prepare( CrawlerMetrics.registerPerSecMetric( context, stormConf, "average_persec", metrics_time_bucket_secs); - // eventCounter MUST be registered before WaitAckCache — the eviction lambda captures it this.eventCounter = CrawlerMetrics.registerCounter( context, stormConf, "counters", metrics_time_bucket_secs); diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java index e203fe351..66d985e70 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/persistence/AbstractSpoutTest.java @@ -33,15 +33,8 @@ import org.opensearch.client.json.jackson.JacksonJsonpMapper; /** - * Unit tests for {@link AbstractSpout#sourceAsMap(JsonData)}. - * - *

Regression coverage for the {@code ClassCastException: JsonStringImpl cannot be cast to - * String} that was thrown by {@link AggregationSpout} and {@link HybridSpout} when the previous - * implementation called {@code hit.source().to(Object.class)}: in the OpenSearch Java client, - * {@code Object.class} is bound to a built-in {@code jsonValueDeserializer} that returns the raw - * Jakarta JSON-P {@code JsonValue} tree (Parsson types) instead of going through the configured - * Jackson mapper. The fix routes deserialisation through {@code Map.class}, which correctly falls - * through to {@link JacksonJsonpMapper} and yields native Java types. + * Unit tests for {@link AbstractSpout#sourceAsMap(JsonData)}: the map holds Java values, not the + * Jakarta JSON-P values of the parsed source. */ class AbstractSpoutTest { @@ -116,14 +109,11 @@ void sourceAsMap_nullSource_returnsEmpty() { } @Test - @DisplayName( - "control: JsonData.to(Object.class) leaks Jakarta JsonValue types — proves the bug") + @DisplayName("control: JsonData.to(Object.class) returns Jakarta JSON-P values") void controlTest_toObjectClassLeaksJsonValue() { - // This test pins the underlying root cause of the original ClassCastException: - // calling JsonData#to(Object.class) bypasses JacksonJsonpMapper and returns a raw - // Jakarta JSON-P JsonValue, NOT a java.lang.String. If a future opensearch-java - // release ever changes this behaviour to align with Map.class, we can simplify the - // production code — until then, AbstractSpout#sourceAsMap must keep using Map.class. + // JsonData#to returns the parsed JSON-P value itself when it is an instance of the + // requested class, as for Object.class and Map.class. + // sourceAsMap therefore asks for LinkedHashMap.class. JsonpMapper mapper = new JacksonJsonpMapper(); JsonData data = parse(mapper, "{\"url\":\"http://example.com/\"}"); @@ -132,8 +122,6 @@ void controlTest_toObjectClassLeaksJsonValue() { // It is a Jakarta JsonObject, not a Java Map: assertInstanceOf(jakarta.json.JsonObject.class, asObject); jakarta.json.JsonObject jsonObject = (jakarta.json.JsonObject) asObject; - // And the "url" entry is a JsonString, not a java.lang.String — exactly the cast that - // blew up at runtime in AggregationSpout before the fix. assertInstanceOf(jakarta.json.JsonString.class, jsonObject.get("url")); } }