#2250 CharsetIdentification: read the content type under the protocol prefix

core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java

@@ -286,11 +286,17 @@ public void execute(Tuple tuple) {
             if (fastCharsetDetection) {
                 charset =
                         CharsetIdentification.getCharsetFast(
-                                metadata, content, maxLengthCharsetDetection);
+                                metadata,
+                                content,
+                                maxLengthCharsetDetection,
+                                protocolMetadataPrefix);
             } else {
                 charset =
                         CharsetIdentification.getCharset(
-                                metadata, content, maxLengthCharsetDetection);
+                                metadata,
+                                content,
+                                maxLengthCharsetDetection,
+                                protocolMetadataPrefix);
             }
 
             LOG.debug(

core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java

@@ -57,10 +57,15 @@ public class CharsetIdentification {
      * guess and if that fails too returns UTF-8. This approach is expected to be faster but maybe
      * less reliable than getCharset.
      *
-     * @since 1.18
+     * @param protocolMetadataPrefix prefix of the response headers in the metadata, including its
+     *     separator (e.g. {@code "protocol."}); empty if they are not prefixed
+     * @since 4.0
      */
     public static String getCharsetFast(
-            final Metadata metadata, final byte[] content, final int maxLengthCharsetDetection) {
+            final Metadata metadata,
+            final byte[] content,
+            final int maxLengthCharsetDetection,
+            final String protocolMetadataPrefix) {
 
         // let's look at the BOM first
         String charset = getCharsetFromBOM(content);
@@ -69,7 +74,7 @@ public static String getCharsetFast(
         }
 
         // then look at what we get from HTTP headers
-        charset = getCharsetFromHTTP(metadata);
+        charset = getCharsetFromHTTP(metadata, protocolMetadataPrefix);
         if (charset != null) {
             return charset;
         }
@@ -92,11 +97,18 @@ public static String getCharsetFast(
     /**
      * Identifies the charset of a document based on the following logic: guess from the
      * ByteOrderMark - else if the same charset is specified in the http headers and the html
-     * metadata then use it - otherwise use ICU's charset detector to make an educated guess and if
-     * that fails too returns UTF-8.
+     * metadata then use it - otherwise use ICU's charset detector to make an educated guess, with
+     * the html metadata or else the http headers as a hint, and if that fails too returns UTF-8.
+     *
+     * @param protocolMetadataPrefix prefix of the response headers in the metadata, including its
+     *     separator (e.g. {@code "protocol."}); empty if they are not prefixed
+     * @since 4.0
      */
     public static String getCharset(
-            Metadata metadata, byte[] content, int maxLengthCharsetDetection) {
+            Metadata metadata,
+            byte[] content,
+            int maxLengthCharsetDetection,
+            String protocolMetadataPrefix) {
 
         // let's look at the BOM first
         String BOMCharset = getCharsetFromBOM(content);
@@ -105,7 +117,7 @@ public static String getCharset(
         }
 
         // then look at what we get from HTTP headers and HTML content
-        String httpCharset = getCharsetFromHTTP(metadata);
+        String httpCharset = getCharsetFromHTTP(metadata, protocolMetadataPrefix);
         String htmlCharset = getCharsetFromMeta(content, maxLengthCharsetDetection);
 
         // both exist and agree
@@ -116,12 +128,7 @@ public static String getCharset(
         }
 
         // let's guess from the text - using a hint or not
-        String hintCharset = null;
-        if (httpCharset != null && htmlCharset == null) {
-            hintCharset = httpCharset;
-        } else if (httpCharset == null && htmlCharset != null) {
-            hintCharset = htmlCharset;
-        }
+        String hintCharset = htmlCharset != null ? htmlCharset : httpCharset;
 
         String textCharset = getCharsetFromText(content, hintCharset, maxLengthCharsetDetection);
         if (textCharset != null) {
@@ -133,8 +140,9 @@ public static String getCharset(
     }
 
     /** Returns the charset declared by the server if any. */
-    private static String getCharsetFromHTTP(Metadata metadata) {
-        return getCharsetFromContentType(metadata.getFirstValue(HttpHeaders.CONTENT_TYPE));
+    private static String getCharsetFromHTTP(Metadata metadata, String protocolMetadataPrefix) {
+        return getCharsetFromContentType(
+                metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, protocolMetadataPrefix));
     }
 
     /** Detects any BOMs and returns the corresponding charset. */

core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java

@@ -22,6 +22,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import org.apache.http.HttpHeaders;
 import org.apache.storm.task.OutputCollector;
 import org.apache.stormcrawler.Constants;
 import org.apache.stormcrawler.Metadata;
@@ -33,6 +34,8 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 
@@ -129,7 +132,10 @@ void charsetDetectionFailureIsReportedAsParseError() throws IOException {
                     .when(
                             () ->
                                     CharsetIdentification.getCharset(
-                                            Mockito.any(), Mockito.any(), Mockito.anyInt()))
+                                            Mockito.any(),
+                                            Mockito.any(),
+                                            Mockito.anyInt(),
+                                            Mockito.any()))
                     .thenThrow(new StackOverflowError());
             parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html");
         }
@@ -365,4 +371,41 @@ void testBaseHrefOutlinkResolution() throws IOException {
                 "https://www.example.be/fr/rayons/stockage-archivage-de-donnees/218725",
                 statusTuples.get(0).get(0));
     }
+
+    /**
+     * The fetcher stores the response headers under protocol.md.prefix, so the charset declared in
+     * the Content-Type has to be read with that prefix. Without it the header is never seen, the
+     * header and the meta charset can never agree, and ICU guesses a UTF-8 page whose detection
+     * window is plain ASCII as ISO-8859-1, see issue #2250.
+     */
+    @ParameterizedTest(name = "detect.charset.fast={0}")
+    @ValueSource(booleans = {false, true})
+    void prefixedContentTypeCharsetIsUsed(boolean fast) throws IOException {
+        Map<String, Object> parserConfig = new HashMap<>();
+        parserConfig.put("protocol.md.prefix", "protocol.");
+        parserConfig.put("detect.charset.maxlength", 2048);
+        parserConfig.put("detect.charset.fast", fast);
+        bolt.prepare(
+                parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+        // an ASCII head longer than the detection window, like most real pages. The default
+        // detection uses the header when it agrees with the meta charset; the fast one returns the
+        // header first, so it is tested without a meta charset, which it would use otherwise
+        String meta = fast ? "" : "<meta charset=\"utf-8\">";
+        String head =
+                meta
+                        + "<meta name=\"description\" content=\""
+                        + "free shipping on every order ".repeat(100)
+                        + "\">";
+        String text = "What’s new: café imotine™";
+        byte[] content =
+                ("<html><head>" + head + "</head><body><p>" + text + "</p></body></html>")
+                        .getBytes(StandardCharsets.UTF_8);
+        Metadata metadata = new Metadata();
+        metadata.setValue("protocol." + HttpHeaders.CONTENT_TYPE, "text/html; charset=UTF-8");
+        parse("http://www.example.com/page.html", content, metadata);
+
+        Assertions.assertEquals(1, output.getEmitted().size());
+        String parsedText = (String) output.getEmitted().get(0).get(3);
+        Assertions.assertTrue(parsedText.contains(text), parsedText);
+    }
 }

core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java

@@ -20,6 +20,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicReference;
+import org.apache.http.HttpHeaders;
 import org.apache.stormcrawler.Metadata;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -56,7 +57,7 @@ private static Throwable detectOnSmallStack(byte[] content, int maxlength)
                         () -> {
                             try {
                                 CharsetIdentification.getCharset(
-                                        new Metadata(), content, maxlength);
+                                        new Metadata(), content, maxlength, "");
                             } catch (Throwable t) {
                                 thrown.set(t);
                             }
@@ -106,7 +107,7 @@ void metaCharsetCutByTheDetectionWindowIsStillRead() {
         page.append("<meta charset=\"").append(DECLARED).append("\">").append(BODY);
         byte[] content = page.toString().getBytes(StandardCharsets.US_ASCII);
 
-        String charset = CharsetIdentification.getCharsetFast(new Metadata(), content, cut);
+        String charset = CharsetIdentification.getCharsetFast(new Metadata(), content, cut, "");
 
         Assertions.assertEquals(DECLARED, charset);
     }
@@ -133,13 +134,59 @@ private static byte[] metaCharsetClosingAfterTheWindow(int quoteOffset) {
     void lookAheadForTheClosingQuoteIsBounded() {
         String within =
                 CharsetIdentification.getCharsetFast(
-                        new Metadata(), metaCharsetClosingAfterTheWindow(LOOKAHEAD - 1), MAXLENGTH);
+                        new Metadata(),
+                        metaCharsetClosingAfterTheWindow(LOOKAHEAD - 1),
+                        MAXLENGTH,
+                        "");
         String beyond =
                 CharsetIdentification.getCharsetFast(
-                        new Metadata(), metaCharsetClosingAfterTheWindow(LOOKAHEAD), MAXLENGTH);
+                        new Metadata(), metaCharsetClosingAfterTheWindow(LOOKAHEAD), MAXLENGTH, "");
 
         Assertions.assertEquals(DECLARED, within);
         Assertions.assertNotEquals(
                 DECLARED, beyond, "a declaration closing beyond the look-ahead is not used");
     }
+
+    /** The prefix the fetcher stores the response headers with, see protocol.md.prefix. */
+    private static final String PREFIX = "protocol.";
+
+    /** The detection window of these tests: the page is plain ASCII within it. */
+    private static final int WINDOW = 2048;
+
+    /**
+     * A UTF-8 page, optionally with a meta charset, whose first non-ASCII byte is past the window.
+     */
+    private static byte[] utf8PageWithAsciiWindow(boolean withMeta) {
+        String meta = withMeta ? "<meta charset=\"utf-8\">" : "";
+        String head =
+                "<meta name=\"description\" content=\"" + "free shipping ".repeat(200) + "\">";
+        return ("<html><head>" + meta + head + "</head><body><p>café</p></body></html>")
+                .getBytes(StandardCharsets.UTF_8);
+    }
+
+    private static Metadata prefixedUtf8ContentType() {
+        Metadata metadata = new Metadata();
+        metadata.setValue(PREFIX + HttpHeaders.CONTENT_TYPE, "text/html; charset=UTF-8");
+        return metadata;
+    }
+
+    /** The fast detection returns the charset of the prefixed Content-Type, see #2250. */
+    @Test
+    void fastDetectionReadsThePrefixedContentType() {
+        String charset =
+                CharsetIdentification.getCharsetFast(
+                        prefixedUtf8ContentType(), utf8PageWithAsciiWindow(false), WINDOW, PREFIX);
+
+        Assertions.assertEquals("UTF-8", charset);
+    }
+
+    /** The prefixed Content-Type agrees with the meta charset, so it is used, see #2250. */
+    @Test
+    void detectionReadsThePrefixedContentType() {
+        String charset =
+                CharsetIdentification.getCharset(
+                        prefixedUtf8ContentType(), utf8PageWithAsciiWindow(true), WINDOW, PREFIX);
+
+        Assertions.assertEquals("UTF-8", charset);
+    }
 }

external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java

@@ -27,8 +27,10 @@
 import org.apache.stormcrawler.parse.ParseData;
 import org.apache.stormcrawler.parse.ParseFilter;
 import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.protocol.playwright.HttpProtocol;
 import org.apache.stormcrawler.util.CharsetIdentification;
+import org.apache.stormcrawler.util.ConfUtils;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.DocumentFragment;
@@ -138,13 +140,16 @@ public class JsRenderingDetector extends ParseFilter {
     private List<String> emptyRootIds = DEFAULT_EMPTY_ROOT_IDS;
     private List<String> requiredMessages = List.of();
     private String skipIfMetadataPresent = HttpProtocol.MD_KEY_END;
+    private String protocolMetadataPrefix = "";
     private boolean recordReason = true;
 
     private final List<Pattern> emptyRootPatterns = new ArrayList<>();
 
     @Override
     public void configure(
             @NotNull final Map<String, Object> stormConf, @NotNull final JsonNode params) {
+        protocolMetadataPrefix =
+                ConfUtils.getString(stormConf, ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, "");
         if (params != null && !params.isMissingNode() && !params.isNull()) {
             if (params.has("metadataKey")) {
                 this.metadataKey = params.get("metadataKey").asText();
@@ -223,7 +228,8 @@ public void filter(
 
     private String detectReason(final String url, final byte[] content, final ParseResult parse) {
         final Metadata md = parse.getOrCreate(url).getMetadata();
-        final String charsetName = CharsetIdentification.getCharsetFast(md, content, -1);
+        final String charsetName =
+                CharsetIdentification.getCharsetFast(md, content, -1, protocolMetadataPrefix);
         java.nio.charset.Charset cs;
         try {
             cs = java.nio.charset.Charset.forName(charsetName);

external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java

@@ -18,8 +18,10 @@
 package org.apache.stormcrawler.protocol.playwright.parsefilter;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
+import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.parse.ParseResult;
 import org.apache.stormcrawler.protocol.playwright.HttpProtocol;
 import org.junit.jupiter.api.Assertions;
@@ -197,4 +199,24 @@ void requiredMessageNoHitDoesNotFlag() throws Exception {
                                 + "</p><a href='/x'>x</a><a href='/y'>y</a><a href='/z'>z</a></body></html>");
         Assertions.assertNull(p.getOrCreate("u").getMetadata().getFirstValue("fetch.with"));
     }
+
+    /**
+     * The page is decoded with the charset of the Content-Type stored under protocol.md.prefix, see
+     * #2250. ISO-8859-15 is not one ICU detects, so without the header the euro sign is read as
+     * another character and the required message is not found.
+     */
+    @Test
+    void decodesWithThePrefixedContentTypeCharset() throws Exception {
+        final String message = "JavaScript required to pay 5 €";
+        final JsRenderingDetector d = new JsRenderingDetector();
+        d.configure(
+                Map.of("protocol.md.prefix", "protocol."),
+                MAPPER.readTree("{\"requiredMessages\":[\"" + message + "\"]}"));
+        final ParseResult parse = new ParseResult();
+        final Metadata md = parse.getOrCreate("u").getMetadata();
+        md.setValue("protocol.Content-Type", "text/html; charset=ISO-8859-15");
+        final String html = "<html><body><p>" + message + "</p></body></html>";
+        d.filter("u", html.getBytes(Charset.forName("ISO-8859-15")), null, parse);
+        Assertions.assertEquals("playwright", md.getFirstValue("fetch.with"));
+    }
 }