Refactor Confluence integration: Update license to MIT, remove requirements.txt, and implement HtmlTextParser for HTML to Markdown conversion. Update dependencies and tests accordingly. (run-llama#20262)

Author: flansen

llama-index-integrations/readers/llama-index-readers-confluence/LICENSE

@@ -2,29 +2,29 @@
 
 import logging
 import os
-import uuid
 import tempfile
+import uuid
+from io import BytesIO
 from typing import Callable, Dict, List, Optional
 from urllib.parse import unquote
 
 import requests
+from llama_index.core.instrumentation import DispatcherSpanMixin, get_dispatcher
 from llama_index.core.readers.base import BaseReader
 from llama_index.core.schema import Document
-from llama_index.core.instrumentation import DispatcherSpanMixin, get_dispatcher
 from retrying import retry
-from io import BytesIO
 
 from .event import (
+    AttachmentFailedEvent,
+    AttachmentProcessedEvent,
+    AttachmentProcessingStartedEvent,
+    AttachmentSkippedEvent,
     FileType,
-    TotalPagesToProcessEvent,
-    PageDataFetchStartedEvent,
     PageDataFetchCompletedEvent,
-    PageSkippedEvent,
+    PageDataFetchStartedEvent,
     PageFailedEvent,
-    AttachmentProcessingStartedEvent,
-    AttachmentProcessedEvent,
-    AttachmentSkippedEvent,
-    AttachmentFailedEvent,
+    PageSkippedEvent,
+    TotalPagesToProcessEvent,
 )
 
 CONFLUENCE_API_TOKEN = "CONFLUENCE_API_TOKEN"
@@ -286,8 +286,7 @@ def load_data(
             != 1
         ):
             raise ValueError(
-                "Must specify exactly one among `space_key`, `page_ids`, `label`, `cql`"
-                " parameters."
+                "Must specify exactly one among `space_key`, `page_ids`, `label`, `cql` parameters."
             )
 
         if cursor and start:
@@ -314,14 +313,9 @@ def load_data(
                 " please use `max_num_results` instead."
             )
 
-        try:
-            import html2text  # type: ignore
-        except ImportError:
-            raise ImportError(
-                "`html2text` package not found, please run `pip install html2text`"
-            )
+        from .html_parser import HtmlTextParser
 
-        text_maker = html2text.HTML2Text()
+        text_maker = HtmlTextParser()
 
         if not start:
             start = 0
@@ -603,7 +597,7 @@ def process_page(self, page, include_attachments, text_maker):
                 except OSError:
                     pass
         else:
-            text = text_maker.handle(page["body"]["export_view"]["value"]) + "".join(
+            text = text_maker.convert(page["body"]["export_view"]["value"]) + "".join(
                 attachment_texts
             )
 
@@ -626,8 +620,7 @@ def process_attachment(self, page_id):
             pass
         except ImportError:
             raise ImportError(
-                "`pytesseract` or `pdf2image` or `Pillow` package not found, please run"
-                " `pip install pytesseract pdf2image Pillow`"
+                "`pytesseract` or `pdf2image` or `Pillow` package not found, please run `pip install pytesseract pdf2image Pillow`"
             )
 
         # depending on setup you may also need to set the correct path for poppler and tesseract
@@ -815,8 +808,7 @@ def process_pdf(self, link):
                 from pdf2image import convert_from_bytes  # type: ignore
             except ImportError:
                 raise ImportError(
-                    "`pytesseract` or `pdf2image` package not found, please run `pip"
-                    " install pytesseract pdf2image`"
+                    "`pytesseract` or `pdf2image` package not found, please run `pip install pytesseract pdf2image`"
                 )
 
         response = self.confluence.request(path=link, absolute=True)
@@ -926,8 +918,7 @@ def process_image(self, link):
             from PIL import Image  # type: ignore
         except ImportError:
             raise ImportError(
-                "`pytesseract` or `Pillow` package not found, please run `pip install"
-                " pytesseract Pillow`"
+                "`pytesseract` or `Pillow` package not found, please run `pip install pytesseract Pillow`"
             )
 
         text = ""
@@ -1166,8 +1157,7 @@ def process_svg(self, link):
             from svglib.svglib import svg2rlg  # type: ignore
         except ImportError:
             raise ImportError(
-                "`pytesseract`, `Pillow`, or `svglib` package not found, please run"
-                " `pip install pytesseract Pillow svglib`"
+                "`pytesseract`, `Pillow`, or `svglib` package not found, please run `pip install pytesseract Pillow svglib`"
             )
 
         response = self.confluence.request(path=link, absolute=True)

llama-index-integrations/readers/llama-index-readers-confluence/llama_index/readers/confluence/base.py

@@ -0,0 +1,21 @@
+class HtmlTextParser:
+    def __init__(self):
+        try:
+            from markdownify import markdownify  # noqa: F401
+        except ImportError:
+            raise ImportError(
+                "`markdownify` package not found, please run `pip install markdownify`"
+            )
+
+    def convert(self, html: str) -> str:
+        from markdownify import markdownify
+
+        if not html:
+            return ""
+
+        return markdownify(
+            html,
+            heading_style="ATX",  # Use # for headings instead of underlines
+            bullets="*",  # Use * for unordered lists
+            strip=["script", "style"],  # Remove script and style tags for security
+        )

llama-index-integrations/readers/llama-index-readers-confluence/llama_index/readers/confluence/html_parser.py

@@ -26,16 +26,15 @@ dev = [
 
 [project]
 name = "llama-index-readers-confluence"
-version = "0.5.0"
+version = "0.6.0"
 description = "llama-index readers confluence integration"
 authors = [{name = "Your Name", email = "[email protected]"}]
-requires-python = ">=3.9,<4.0"
+requires-python = ">=3.9,<3.14"
 readme = "README.md"
-license = "GPL-3.0-or-later"
+license = "MIT"
 maintainers = [{name = "zywilliamli"}]
 dependencies = [
     "atlassian-python-api>=3.41.9,<5",
-    "html2text>=2024.2.26,<2025",
     "pytesseract>=0.3.10,<0.4",
     "pdf2image>=1.17.0,<2",
     "pillow>=10.2.0,<11",
@@ -44,6 +43,7 @@ dependencies = [
     "svglib>=1.5,<1.6",
     "retrying>=1.3.4,<2",
     "llama-index-core>=0.13.0,<0.15",
+    "markdownify>=1.2.0,<2.0.0",
 ]
 
 [tool.codespell]