#2198 preserve query strings containing slashes

core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java

@@ -333,23 +333,30 @@ public int compare(NameValuePair p1, NameValuePair p2) {
             };
 
     /**
-     * A common error to find is a query string that starts with an & instead of a ? This will fix
-     * that error. So http://foo.com&a=b will be changed to http://foo.com?a=b.
+     * Repairs a query string that starts with an & instead of a ?. The query check ignores the
+     * fragment, and the repair only checks the last path segment so an & in an earlier path segment
+     * is not mistaken for a query separator.
      *
      * @param urlToFilter the url to filter
      * @return corrected url
      */
     private String unmangleQueryString(String urlToFilter) {
-        String[] pathElements = urlToFilter.split("/");
+        int fragmentStart = urlToFilter.indexOf('#');
+        String fragment = fragmentStart == -1 ? "" : urlToFilter.substring(fragmentStart);
+        String url = fragmentStart == -1 ? urlToFilter : urlToFilter.substring(0, fragmentStart);
+        if (url.indexOf('?') != -1) {
+            return urlToFilter;
+        }
+
+        String[] pathElements = url.split("/", -1);
         final String lastPathElement = pathElements[pathElements.length - 1];
         int firstAmp = lastPathElement.indexOf('&');
         if (firstAmp == -1) {
             return urlToFilter;
         }
-        int firstQuestionMark = lastPathElement.indexOf('?');
-        if (firstQuestionMark == -1 && lastPathElement.indexOf("=") > 0) {
+        if (lastPathElement.indexOf("=") > 0) {
             pathElements[pathElements.length - 1] = lastPathElement.replaceFirst("&", "?");
-            return String.join("/", pathElements);
+            return String.join("/", pathElements) + fragment;
         }
         return urlToFilter;
     }

core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java

@@ -212,6 +212,67 @@ void testFixMangledQueryString() throws MalformedURLException {
         assertEquals(expectedResult, normalizedUrl, "Failed to filter query string");
     }
 
+    @Test
+    void testDoesNotMangleQueryWhenValueContainsSlashes() throws MalformedURLException {
+        URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance));
+        URL testSourceUrl = URLUtil.toURL("http://a.example/");
+        String inputUrl = "http://a.example/article.pl?sid=01/10/23/1816257&mode=thread&tid=107";
+        assertEquals(
+                inputUrl,
+                urlFilter.filter(testSourceUrl, new Metadata(), inputUrl),
+                "An existing query must not be unmangled because a value contains slashes");
+
+        inputUrl = "http://a.example/p?u=http://b.example/x&z=1";
+        assertEquals(
+                inputUrl,
+                urlFilter.filter(testSourceUrl, new Metadata(), inputUrl),
+                "An embedded URL in a query value must remain intact");
+    }
+
+    @Test
+    void testUnmanglesQueryBeforeFragmentQuestionMark() throws MalformedURLException {
+        URLFilter urlFilter = createFilter(false, false);
+        URL testSourceUrl = URLUtil.toURL("http://a.example/");
+        String inputUrl = "http://a.example/p&a=b#frag?x";
+        assertEquals(
+                "http://a.example/p?a=b#frag?x",
+                urlFilter.filter(testSourceUrl, new Metadata(), inputUrl),
+                "A question mark in the fragment must not suppress query repair");
+    }
+
+    @Test
+    void testDoesNotDropTrailingSlashWhenUnmanglingWithFragment() throws MalformedURLException {
+        URLFilter urlFilter = createFilter(false, false);
+        URL testSourceUrl = URLUtil.toURL("http://a.example/");
+        String inputUrl = "http://a.example/x&a=b/#frag";
+        assertEquals(
+                inputUrl,
+                urlFilter.filter(testSourceUrl, new Metadata(), inputUrl),
+                "A trailing slash before a fragment must not be mistaken for a query");
+    }
+
+    @Test
+    void testDoesNotDropTrailingSlashWhenUnmanglingByDefault() throws MalformedURLException {
+        URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance));
+        URL testSourceUrl = URLUtil.toURL("http://a.example/");
+        String inputUrl = "http://a.example/x&a=b/";
+        assertEquals(
+                inputUrl,
+                urlFilter.filter(testSourceUrl, new Metadata(), inputUrl),
+                "A trailing slash must not be removed during default normalization");
+    }
+
+    @Test
+    void testRemovesQueryElementWhenValueContainsSlashes() throws MalformedURLException {
+        URLFilter urlFilter = createFilter(List.of("utm_source"));
+        URL testSourceUrl = URLUtil.toURL("http://a.example/");
+        String inputUrl = "http://a.example/login/?next=/a/b/&utm_source=ref&z=1";
+        assertEquals(
+                "http://a.example/login/?next=%2Fa%2Fb%2F&z=1",
+                urlFilter.filter(testSourceUrl, new Metadata(), inputUrl),
+                "Query elements following a slash-containing value must remain removable");
+    }
+
     @Test
     void testProperURLEncodingWithoutQueryParameter() throws MalformedURLException {
         URLFilter urlFilter = createFilter(queryParamsToFilter);