feat: add sitemap loader to admin frontend (#21)

* feat: add Confluence integration with configurable parameters for document loading

* chore: update submodules to latest main

* feat: enable MinIO feature in Helm chart configuration and remove default max pages limit in document upload

* feat: add pytest configuration and update testing setup across multiple components

* Update frontend/libs/i18n/admin/en.json

Co-authored-by: Copilot <[email protected]>

* feat: enhance Confluence upload form with additional input fields and labels

* feat: add sitemap upload functionality with configuration options and error handling

* feat: add header template support for sitemap upload with JSON validation

* chore: update .gitignore to include todo files, remove unused Confluence configuration from DocumentUploadContainer, and update subproject reference in rag-core-library

* chore: update subproject commit reference in rag-core-library

* chore: update subproject commit reference in rag-core-library

* chore: update subproject commit reference in rag-core-library

* chore: update submodules to latest main

* feat: update dependencies and add new packages

- Added `fake-useragent` version 2.2.0 to `document-extractor/poetry.lock`.
- Introduced `googleapis-common-protos` version 1.70.0 and updated dependencies in `rag-backend/poetry.lock`.
- Added `importlib-metadata` version 8.7.0 with dependencies in `rag-backend/poetry.lock`.
- Updated `langfuse` to version 3.0.0 in `rag-backend/poetry.lock` and adjusted its dependencies.
- Added OpenTelemetry packages (`opentelemetry-api`, `opentelemetry-exporter-otlp`, `opentelemetry-sdk`, etc.) with version 1.34.1 in `rag-backend/poetry.lock`.
- Downgraded `protobuf` to version 5.29.5 in `rag-backend/poetry.lock`.
- Added `zipp` version 3.23.0 in `rag-backend/poetry.lock`.
- Updated submodules for `rag-core-library` and `rag-infrastructure`.

* chore: update submodules to latest main

* Update subproject commits for rag-core-library and rag-infrastructure

* chore: update submodules to latest main

* chore: update subproject commit for rag-core-library

* Update frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue

Co-authored-by: Copilot <[email protected]>

* Update frontend/libs/admin-app/data-access/+state/documents.store.ts

Co-authored-by: Copilot <[email protected]>

* Update DocumentUploadContainer.vue

* chore: update submodule

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Copilot <[email protected]>

Author: 3 people

.gitignore

@@ -11,7 +11,8 @@ auth
 # Node Modules
 node_modules/
 
-
+**/.notebooks
+**/todo*.md
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

admin-backend/poetry.lock

@@ -3,13 +3,14 @@ import { ref } from 'vue';
 import { DocumentModel } from "../../models/document.model.ts";
 import { ErrorType } from "../../models/error-type";
 import { UploadedDocument, mapToUploadDocument } from "../../models/uploaded-document.model";
-import { DocumentAPI, ConfluenceConfig } from "../document.api";
+import { ConfluenceConfig, DocumentAPI, SitemapConfig } from "../document.api";
 
 export const useDocumentsStore = defineStore('chat', () => {
     const uploadedDocuments = ref<UploadedDocument[]>([]);
     const allDocuments = ref<DocumentModel[]>();
     const error = ref<ErrorType | null>(null);
     const isLoadingConfluence = ref(false);
+    const isLoadingSitemap = ref(false);
 
     function updateUploadedDocumentData(documentId: string, data: Partial<UploadedDocument>) {
         const document = uploadedDocuments.value.find((d: UploadedDocument) => d.id === documentId);
@@ -75,6 +76,29 @@ export const useDocumentsStore = defineStore('chat', () => {
       }
     };
 
+    const loadSitemap = async (config: SitemapConfig) => {
+      isLoadingSitemap.value = true;
+      error.value = null;
+      try {
+        // provide sitemap configuration from frontend
+        await DocumentAPI.loadSitemap(config);
+        await loadDocuments(); // Refresh the document list after uploading
+      } catch(err) {
+        if (err.response && err.response.status === 501) {
+          error.value = "sitemap_not_configured";
+          console.error("Sitemap loader is not configured.");
+        } else if (err.response && err.response.status === 423) {
+          error.value = "sitemap_locked";
+          console.error("Sitemap loader returned a warning.");
+        } else {
+          error.value = "sitemap";
+          console.error(err);
+        }
+      } finally {
+        isLoadingSitemap.value = false;
+      }
+    };
+
     const uploadDocuments = async (files: File[]) => {
         try {
             const uploads = files.map(uploadDocument);
@@ -103,5 +127,5 @@ export const useDocumentsStore = defineStore('chat', () => {
         uploadedDocuments.value = uploadedDocuments.value.filter(o => o.id !== documentId);
     };
 
-    return {removeUploadedDocument, uploadDocuments, loadDocuments, deleteDocument, loadConfluence, allDocuments, uploadedDocuments, error};
+    return {removeUploadedDocument, uploadDocuments, loadDocuments, deleteDocument, loadConfluence, loadSitemap, allDocuments, uploadedDocuments, error, isLoadingSitemap};
 });

document-extractor/poetry.lock

@@ -16,6 +16,14 @@ export interface ConfluenceConfig {
   name: string;
 }
 
+// sitemap configuration interface
+export interface SitemapConfig {
+  webPath: string;
+  filterUrls: string;
+  headerTemplate: string;
+  name: string;
+}
+
 export class DocumentAPI {
     static async loadDocuments(): Promise<DocumentModel[]> {
         try {
@@ -62,6 +70,46 @@ export class DocumentAPI {
         }
     }
 
+    static async loadSitemap(config: SitemapConfig): Promise<void> {
+        try {
+            // convert config to list of key/value items for backend
+            const payload = [
+                { key: 'web_path', value: config.webPath }
+            ];
+
+            // add filter_urls only if provided
+            if (config.filterUrls && config.filterUrls.trim()) {
+                // Convert multiline string to array and filter out empty lines
+                const filterUrlsArray = config.filterUrls
+                    .split('\n')
+                    .map(url => url.trim())
+                    .filter(url => url.length > 0);
+
+                if (filterUrlsArray.length > 0) {
+                    payload.push({ key: 'filter_urls', value: JSON.stringify(filterUrlsArray) });
+                }
+            }
+
+            // add header_template only if provided
+            if (config.headerTemplate && config.headerTemplate.trim()) {
+                try {
+                    // Validate JSON format
+                    JSON.parse(config.headerTemplate);
+                    payload.push({ key: 'header_template', value: config.headerTemplate });
+                } catch (jsonError) {
+                    throw new Error('Header template must be valid JSON format');
+                }
+            }
+
+            // include required query parameters
+            await axios.post<void>('/upload_source', payload, {
+                params: { source_type: 'sitemap', name: config.name }
+            });
+        } catch(error) {
+            this.handleError(error);
+        }
+    }
+
     static async deleteDocument(documentId: string): Promise<void> {
         try {
             await axios.delete<void>(`/delete_document/${documentId}`);

frontend/libs/admin-app/data-access/+state/documents.store.ts

@@ -13,7 +13,7 @@ const fileInputRef = ref<HTMLInputElement>();
 const uploadedDocuments = computed((): UploadedDocument[] => store.uploadedDocuments);
 const isInvalidFileType = ref(false);
 const allowedFileTypes = ['application/pdf', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'text/xml'];
-const uploadMethod = ref<'file' | 'confluence'>('file');
+const uploadMethod = ref<'file' | 'confluence' | 'sitemap'>('file');
 
 
 // confluence configuration refs
@@ -23,6 +23,12 @@ const confluenceToken = ref('');
 const confluenceUrl = ref('');
 const maxPages = ref<number>();
 
+// sitemap configuration refs
+const sitemapName = ref('');
+const sitemapWebPath = ref('');
+const sitemapFilterUrls = ref('');
+const sitemapHeaderTemplate = ref('');
+
 const error = computed(() => store.error);
 
 const uploadDocuments = (files: File[]) => {
@@ -73,6 +79,16 @@ const handleConfluenceUpload = () => {
     });
 }
 
+const handleSitemapUpload = () => {
+    // send configured parameters to backend
+    store.loadSitemap({
+        name: sitemapName.value,
+        webPath: sitemapWebPath.value,
+        filterUrls: sitemapFilterUrls.value,
+        headerTemplate: sitemapHeaderTemplate.value
+    });
+}
+
 const clearError = () => {
     store.error = null
 }
@@ -91,6 +107,12 @@ const getErrorMessage = (errorType: string) => {
             return t('documents.confluenceNotConfigured');
         case 'confluence_locked':
             return t('documents.confluenceLocked');
+        case 'sitemap':
+            return t('documents.sitemapError');
+        case 'sitemap_not_configured':
+            return t('documents.sitemapNotConfigured');
+        case 'sitemap_locked':
+            return t('documents.sitemapLocked');
         default:
             return t('documents.unknownError');
     }
@@ -124,6 +146,10 @@ const getErrorMessage = (errorType: string) => {
                     @click="uploadMethod = 'confluence'">
                     {{ t('documents.confluenceUpload') }}
                 </a>
+                <a class="tab" :class="{'tab-active': uploadMethod === 'sitemap'}"
+                    @click="uploadMethod = 'sitemap'">
+                    {{ t('documents.sitemapUpload') }}
+                </a>
             </div>
 
             <!-- File upload area -->
@@ -144,7 +170,7 @@ const getErrorMessage = (errorType: string) => {
             </div>
 
             <!-- Confluence load area -->
-            <div v-else
+            <div v-else-if="uploadMethod === 'confluence'"
                 class="flex flex-col m-auto justify-center items-center w-full h-112 bg-base-100 rounded-box border border-base-300">
                 <div class="flex flex-col justify-center items-center pt-5 pb-6">
                     <GlobeAltIcon class="w-10 h-10 mb-4 text-accent-content" />
@@ -169,6 +195,30 @@ const getErrorMessage = (errorType: string) => {
                 </div>
             </div>
 
+            <!-- Sitemap load area -->
+            <div v-else-if="uploadMethod === 'sitemap'"
+                class="flex flex-col m-auto justify-center items-center w-full h-112 bg-base-100 rounded-box border border-base-300">
+                <div class="flex flex-col justify-center items-center pt-5 pb-6">
+                    <GlobeAltIcon class="w-10 h-10 mb-4 text-accent-content" />
+                    <p class="mb-1 font-bold">{{ t('documents.sitemapLoadTitle') }}</p>
+                    <!-- configuration inputs -->
+                    <div class="space-y-2 mb-4 w-full max-w-sm">
+                      <label for="sitemapName" class="sr-only">Sitemap Name</label>
+                      <input id="sitemapName" v-model="sitemapName" type="text" placeholder="Name" class="input input-bordered w-full" required/>
+                      <label for="sitemapWebPath" class="sr-only">Sitemap URL</label>
+                      <input v-model="sitemapWebPath" type="url" placeholder="Sitemap URL (required)" class="input input-bordered w-full" required />
+                      <label for="sitemapFilterUrls" class="sr-only">Filter URLs</label>
+                      <textarea v-model="sitemapFilterUrls" placeholder="Filter URLs (optional) - one regex pattern per line" class="textarea textarea-bordered w-full" rows="3"></textarea>
+                      <label for="sitemapHeaderTemplate" class="sr-only">Headers JSON</label>
+                      <textarea v-model="sitemapHeaderTemplate" placeholder="Headers (optional) - JSON format: {&quot;Authorization&quot;: &quot;Bearer token&quot;}" class="textarea textarea-bordered w-full" rows="2"></textarea>
+                    </div>
+                    <p class="text-xs opacity-50 mb-4">{{ t('documents.sitemapLoadDescription') }}</p>
+                    <button class="btn btn-sm btn-accent" @click="handleSitemapUpload">
+                        {{ t('documents.loadSitemap') }}
+                    </button>
+                </div>
+            </div>
+
             <!-- Uploaded documents -->
             <div class="mx-auto mt-4 w-full">
                 <div class="mb-4" v-for="uploadDocument in uploadedDocuments" :key="uploadDocument.id">

frontend/libs/admin-app/data-access/document.api.ts

@@ -15,6 +15,10 @@
     "confluenceLoadTitle": "Load all Confluence pages from a space",
     "confluenceLoadDescription": "Click the button below to load pages from Confluence",
     "loadConfluence": "Load Confluence",
+    "sitemapUpload": "Sitemap",
+    "sitemapLoadTitle": "Load content from a sitemap",
+    "sitemapLoadDescription": "Enter a sitemap URL to extract and load content from all linked pages",
+    "loadSitemap": "Load Sitemap",
     "select": "Select",
     "chat": "Start chat",
     "uploadDocumentFailed": "Upload failed",
@@ -25,6 +29,9 @@
     "confluenceError": "Failed to load from Confluence",
     "confluenceNotConfigured": "Confluence is not configured",
     "confluenceLocked": "Confluence Loader is busy. Please try again later",
+    "sitemapError": "Failed to load from sitemap",
+    "sitemapNotConfigured": "Sitemap is not configured",
+    "sitemapLocked": "Sitemap Loader is busy. Please try again later",
     "unknownError": "An unknown error occurred"
   }
 }