From b5cc1794d7b4050e6ec5c6a8ba8f3b61e7556811 Mon Sep 17 00:00:00 2001 From: sean0407 <22557931+sean0407@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:34:42 +0800 Subject: [PATCH 1/4] fix: preserve query strings containing slashes --- .../filtering/basic/BasicURLNormalizer.java | 7 +++++++ .../filtering/BasicURLNormalizerTest.java | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index 4e14119e5..f138c2d37 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -340,6 +340,13 @@ public int compare(NameValuePair p1, NameValuePair p2) { * @return corrected url */ private String unmangleQueryString(String urlToFilter) { + int fragmentStart = urlToFilter.indexOf('#'); + String urlBeforeFragment = + fragmentStart == -1 ? urlToFilter : urlToFilter.substring(0, fragmentStart); + if (urlBeforeFragment.indexOf('?') != -1) { + return urlToFilter; + } + String[] pathElements = urlToFilter.split("/"); final String lastPathElement = pathElements[pathElements.length - 1]; int firstAmp = lastPathElement.indexOf('&'); diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 5df8b97a7..8b124f9d5 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -212,6 +212,24 @@ void testFixMangledQueryString() throws MalformedURLException { assertEquals(expectedResult, normalizedUrl, "Failed to filter query string"); } + @Test + void testDoesNotMangleQueryWhenValueContainsSlashes() throws MalformedURLException { + URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = + "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107"; + assertEquals( + inputUrl, + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "An existing query must not be unmangled because a value contains slashes"); + + inputUrl = "http://a.example/p?u=http://b.example/x&z=1"; + assertEquals( + inputUrl, + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "An embedded URL in a query value must remain intact"); + } + @Test void testProperURLEncodingWithoutQueryParameter() throws MalformedURLException { URLFilter urlFilter = createFilter(queryParamsToFilter); From aa5a4b909915e07cd992e002b105ee7c7d3b5f5f Mon Sep 17 00:00:00 2001 From: Sean Li <22557931+sean0407@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:28:29 +0800 Subject: [PATCH 2/4] fix: preserve query fragment during repair --- .../filtering/basic/BasicURLNormalizer.java | 18 ++++++------- .../filtering/BasicURLNormalizerTest.java | 25 +++++++++++++++++-- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index f138c2d37..3c7b1546c 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -333,30 +333,30 @@ public int compare(NameValuePair p1, NameValuePair p2) { }; /** - * A common error to find is a query string that starts with an & instead of a ? This will fix - * that error. So http://foo.com&a=b will be changed to http://foo.com?a=b. + * Repairs a query string that starts with an & instead of a ?. The query check ignores the + * fragment, and the repair only checks the last path segment so an & in an earlier path segment + * is not mistaken for a query separator. * * @param urlToFilter the url to filter * @return corrected url */ private String unmangleQueryString(String urlToFilter) { int fragmentStart = urlToFilter.indexOf('#'); - String urlBeforeFragment = - fragmentStart == -1 ? urlToFilter : urlToFilter.substring(0, fragmentStart); - if (urlBeforeFragment.indexOf('?') != -1) { + String fragment = fragmentStart == -1 ? "" : urlToFilter.substring(fragmentStart); + String url = fragmentStart == -1 ? urlToFilter : urlToFilter.substring(0, fragmentStart); + if (url.indexOf('?') != -1) { return urlToFilter; } - String[] pathElements = urlToFilter.split("/"); + String[] pathElements = url.split("/"); final String lastPathElement = pathElements[pathElements.length - 1]; int firstAmp = lastPathElement.indexOf('&'); if (firstAmp == -1) { return urlToFilter; } - int firstQuestionMark = lastPathElement.indexOf('?'); - if (firstQuestionMark == -1 && lastPathElement.indexOf("=") > 0) { + if (lastPathElement.indexOf("=") > 0) { pathElements[pathElements.length - 1] = lastPathElement.replaceFirst("&", "?"); - return String.join("/", pathElements); + return String.join("/", pathElements) + fragment; } return urlToFilter; } diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 8b124f9d5..1c89677aa 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -216,8 +216,7 @@ void testFixMangledQueryString() throws MalformedURLException { void testDoesNotMangleQueryWhenValueContainsSlashes() throws MalformedURLException { URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); URL testSourceUrl = URLUtil.toURL("http://a.example/"); - String inputUrl = - "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107"; + String inputUrl = "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107"; assertEquals( inputUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), @@ -230,6 +229,28 @@ void testDoesNotMangleQueryWhenValueContainsSlashes() throws MalformedURLExcepti "An embedded URL in a query value must remain intact"); } + @Test + void testUnmanglesQueryBeforeFragmentQuestionMark() throws MalformedURLException { + URLFilter urlFilter = createFilter(false, false); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = "http://a.example/p&a=b#frag?x"; + assertEquals( + "http://a.example/p?a=b#frag?x", + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "A question mark in the fragment must not suppress query repair"); + } + + @Test + void testRemovesQueryElementWhenValueContainsSlashes() throws MalformedURLException { + URLFilter urlFilter = createFilter(List.of("utm_source")); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = "http://a.example/login/?next=/a/b/&utm_source=ref&z=1"; + assertEquals( + "http://a.example/login/?next=%2Fa%2Fb%2F&z=1", + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "Query elements following a slash-containing value must remain removable"); + } + @Test void testProperURLEncodingWithoutQueryParameter() throws MalformedURLException { URLFilter urlFilter = createFilter(queryParamsToFilter); From 8812ee8e3b0568855a2507a3b5d7b9d324a591cb Mon Sep 17 00:00:00 2001 From: Sean Li <22557931+sean0407@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:24:02 +0800 Subject: [PATCH 3/4] fix: preserve trailing slash during query repair --- .../filtering/basic/BasicURLNormalizer.java | 2 +- .../filtering/BasicURLNormalizerTest.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index 3c7b1546c..400f00964 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -348,7 +348,7 @@ private String unmangleQueryString(String urlToFilter) { return urlToFilter; } - String[] pathElements = url.split("/"); + String[] pathElements = url.split("/", -1); final String lastPathElement = pathElements[pathElements.length - 1]; int firstAmp = lastPathElement.indexOf('&'); if (firstAmp == -1) { diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 1c89677aa..1fab469c9 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -240,6 +240,28 @@ void testUnmanglesQueryBeforeFragmentQuestionMark() throws MalformedURLException "A question mark in the fragment must not suppress query repair"); } + @Test + void testDoesNotDropTrailingSlashWhenUnmanglingWithFragment() throws MalformedURLException { + URLFilter urlFilter = createFilter(false, false); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = "http://a.example/x&a=b/#frag"; + assertEquals( + inputUrl, + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "A trailing slash before a fragment must not be mistaken for a query"); + } + + @Test + void testDoesNotDropTrailingSlashWhenUnmanglingByDefault() throws MalformedURLException { + URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + String inputUrl = "http://a.example/x&a=b/"; + assertEquals( + inputUrl, + urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), + "A trailing slash must not be removed during default normalization"); + } + @Test void testRemovesQueryElementWhenValueContainsSlashes() throws MalformedURLException { URLFilter urlFilter = createFilter(List.of("utm_source")); From 70a050faba531bb45d4472d85cb0457362da78f2 Mon Sep 17 00:00:00 2001 From: Sean Date: Sun, 25 Oct 2026 07:23:47 +0800 Subject: [PATCH 4/4] Improve unmangled query repair checks --- .../filtering/basic/BasicURLNormalizer.java | 14 +++-- .../filtering/BasicURLNormalizerTest.java | 56 ++++++++++++------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index 400f00964..632752cfb 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -341,6 +341,10 @@ public int compare(NameValuePair p1, NameValuePair p2) { * @return corrected url */ private String unmangleQueryString(String urlToFilter) { + if (urlToFilter.indexOf('&') == -1) { + return urlToFilter; + } + int fragmentStart = urlToFilter.indexOf('#'); String fragment = fragmentStart == -1 ? "" : urlToFilter.substring(fragmentStart); String url = fragmentStart == -1 ? urlToFilter : urlToFilter.substring(0, fragmentStart); @@ -348,15 +352,13 @@ private String unmangleQueryString(String urlToFilter) { return urlToFilter; } - String[] pathElements = url.split("/", -1); - final String lastPathElement = pathElements[pathElements.length - 1]; - int firstAmp = lastPathElement.indexOf('&'); + int lastSlash = url.lastIndexOf('/'); + int firstAmp = url.indexOf('&', lastSlash + 1); if (firstAmp == -1) { return urlToFilter; } - if (lastPathElement.indexOf("=") > 0) { - pathElements[pathElements.length - 1] = lastPathElement.replaceFirst("&", "?"); - return String.join("/", pathElements) + fragment; + if (url.indexOf('=', firstAmp + 1) > firstAmp) { + return url.substring(0, firstAmp) + '?' + url.substring(firstAmp + 1) + fragment; } return urlToFilter; } diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 1fab469c9..4f050c4c6 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -35,6 +35,8 @@ import org.apache.stormcrawler.util.URLUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; /** * Utility class which encapsulates the filtering of URLs based on the hostname or domain of the @@ -212,21 +214,20 @@ void testFixMangledQueryString() throws MalformedURLException { assertEquals(expectedResult, normalizedUrl, "Failed to filter query string"); } - @Test - void testDoesNotMangleQueryWhenValueContainsSlashes() throws MalformedURLException { + @ParameterizedTest + @CsvSource({ + "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107", + "http://a.example/p?u=http://b.example/x&z=1", + "http://a.example/login/?next=/a/b/&utm_source=ref&z=1" + }) + void testDoesNotMangleQueryWhenValueContainsSlashes(String inputUrl) + throws MalformedURLException { URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); URL testSourceUrl = URLUtil.toURL("http://a.example/"); - String inputUrl = "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107"; assertEquals( inputUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), "An existing query must not be unmangled because a value contains slashes"); - - inputUrl = "http://a.example/p?u=http://b.example/x&z=1"; - assertEquals( - inputUrl, - urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), - "An embedded URL in a query value must remain intact"); } @Test @@ -240,26 +241,39 @@ void testUnmanglesQueryBeforeFragmentQuestionMark() throws MalformedURLException "A question mark in the fragment must not suppress query repair"); } - @Test - void testDoesNotDropTrailingSlashWhenUnmanglingWithFragment() throws MalformedURLException { - URLFilter urlFilter = createFilter(false, false); + @ParameterizedTest + @CsvSource({ + "false, http://a.example/x&a=b/#frag", + "true, http://a.example/x&a=b/" + }) + void testDoesNotDropTrailingSlashWhenUnmangling(boolean removeAnchor, String inputUrl) + throws MalformedURLException { + URLFilter urlFilter = createFilter(removeAnchor, true, List.of()); URL testSourceUrl = URLUtil.toURL("http://a.example/"); - String inputUrl = "http://a.example/x&a=b/#frag"; assertEquals( inputUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), - "A trailing slash before a fragment must not be mistaken for a query"); + "The URL must remain unchanged when the last path segment has no query parameters"); + } + + @ParameterizedTest + @CsvSource({ + "http://a.example/p&a=b, http://a.example/p?a=b", + "http://a.example/p&a=b&c=d, http://a.example/p?a=b&c=d" + }) + void testRepairsMangledQueryString(String inputUrl, String expectedUrl) + throws MalformedURLException { + URLFilter urlFilter = createFilter(false, true, List.of()); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + assertEquals(expectedUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl)); } @Test - void testDoesNotDropTrailingSlashWhenUnmanglingByDefault() throws MalformedURLException { - URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance)); + void testDoesNotRewriteAmpersandInFragment() throws MalformedURLException { + URLFilter urlFilter = createFilter(false, true, List.of()); URL testSourceUrl = URLUtil.toURL("http://a.example/"); - String inputUrl = "http://a.example/x&a=b/"; - assertEquals( - inputUrl, - urlFilter.filter(testSourceUrl, new Metadata(), inputUrl), - "A trailing slash must not be removed during default normalization"); + String inputUrl = "http://a.example/p#a&b=c"; + assertEquals(inputUrl, urlFilter.filter(testSourceUrl, new Metadata(), inputUrl)); } @Test