FIX Web Crawler doesn't include search params in URLs (#2300)

FlowiseAI · May 2, 2024 · 2254d16 · 2254d16
1 parent c5e06bc
commit 2254d16
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts
@@ -308,7 +308,7 @@ function getURLsFromHTML(htmlBody: string, baseURL: string): string[] {
  */
 function normalizeURL(urlString: string): string {
     const urlObj = new URL(urlString)
-    const hostPath = urlObj.hostname + urlObj.pathname
+    const hostPath = urlObj.hostname + urlObj.pathname + urlObj.search
     if (hostPath.length > 0 && hostPath.slice(-1) == '/') {
         // handling trailing slash
         return hostPath.slice(0, -1)