03_download.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92c3dff2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| default_exp download"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d533800",
   "metadata": {},
   "source": [
    "# Download helpers\n",
    "\n",
    "- Download and process LLM-ready documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e58d8c43",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "from fastcore.utils import *\n",
    "from httpx import get\n",
    "from fastcore.meta import delegates\n",
    "from urllib.parse import urlparse, urljoin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30199708",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import Markdown,HTML\n",
    "from fastcore.test import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95c4cab1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def clean_md(text, rm_comments=True, rm_details=True):\n",
    "    \"Remove comments and `<details>` sections from `text`\"\n",
    "    if rm_comments: text = re.sub(r'\\n?<!--.*?-->\\n?', '', text, flags=re.DOTALL)\n",
    "    if rm_details: text = re.sub(r'\\n?<details>.*?</details>\\n?', '', text, flags=re.DOTALL)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f3d5c69",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@delegates(get)\n",
    "def read_md(url, rm_comments=True, rm_details=True, **kwargs):\n",
    "    \"Read text from `url` and clean with `clean_docs`\"\n",
    "    return clean_md(get(url, **kwargs).text, rm_comments=rm_comments, rm_details=rm_details)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "478d5508",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdurl = 'https://claudette.answer.ai/index.html.md'\n",
    "md = read_md(mdurl)\n",
    "# Markdown(md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8d61937",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def html2md(s:str, ignore_links=True):\n",
    "    \"Convert `s` from HTML to markdown\"\n",
    "    import html2text\n",
    "    o = html2text.HTML2Text(bodywidth=5000)\n",
    "    o.ignore_links = ignore_links\n",
    "    o.mark_code = True\n",
    "    o.ignore_images = True\n",
    "    return o.handle(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e897053",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def read_html(url, # URL to read\n",
    "              sel=None, # Read only outerHTML of CSS selector `sel`\n",
    "              rm_comments=True, # Removes HTML comments\n",
    "              rm_details=True, # Removes `<details>` tags\n",
    "              multi=False, # Get all matches to `sel` or first one  \n",
    "              wrap_tag=None, #If multi, each selection wrapped with <wrap_tag>content</wrap_tag>\n",
    "              ignore_links=True,\n",
    "             ): # Cleaned markdown\n",
    "    \"Get `url`, optionally selecting CSS selector `sel`, and convert to clean markdown\"\n",
    "    page = get(url).text\n",
    "    if sel:\n",
    "        from bs4 import BeautifulSoup\n",
    "        soup = BeautifulSoup(page, 'html.parser')\n",
    "        if multi:\n",
    "            page = [str(el) for el in soup.select(sel)]\n",
    "            if not wrap_tag: page = \"\\n\".join(page)\n",
    "        else: page = str(soup.select_one(sel))\n",
    "    mds = map(lambda x: clean_md(html2md(x, ignore_links=ignore_links), rm_comments, rm_details=rm_details), tuplify(page))\n",
    "    if wrap_tag: return '\\n'.join([f\"\\n<{wrap_tag}>\\n{o}</{wrap_tag}>\\n\" for o in mds])\n",
    "    else: return'\\n'.join(mds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d07c687",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test single class selector\n",
    "listings = read_html('https://www.answer.ai/', sel='.listing-description')\n",
    "assert len(listings) < 500\n",
    "\n",
    "# Test multi class selector\n",
    "listings = read_html('https://www.answer.ai/', sel='.listing-description', multi=True)\n",
    "assert len(listings) > 1000 # returns more than single so selecting multi\n",
    "\n",
    "# Test multi_wrap_tag\n",
    "listings = read_html('https://www.answer.ai/', sel='.listing-description', multi=True, wrap_tag='document')\n",
    "assert '<document>' in listings and '</document>' in listings "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20188898",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[My experience learning GPU programming, and implementing a new GPU education app in the process](./posts/2025-03-17-gpu-programming-scratch.html)\\n\\n'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "read_html('https://www.answer.ai/', sel='.listing-description', ignore_links=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7406a52d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test tag css selectors\n",
    "assert len(read_html('https://www.answer.ai/', sel='div.listing-description', multi=True)) > 1000\n",
    "assert len(read_html('https://www.answer.ai/', sel='div', multi=True)) > 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f25e767",
   "metadata": {},
   "outputs": [],
   "source": [
    "htmlurl = 'https://hypermedia.systems/hypermedia-a-reintroduction/'\n",
    "hmd = read_html(htmlurl)\n",
    "assert len(hmd) > 100\n",
    "# Markdown(hmd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "066b5532",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def get_llmstxt(url, optional=False, n_workers=None):\n",
    "    \"Get llms.txt file from and expand it with `llms_txt.create_ctx()`\"\n",
    "    if not url.endswith('llms.txt'): return None\n",
    "    import llms_txt\n",
    "    resp = get(url)\n",
    "    if resp.status_code!=200: return None\n",
    "    return llms_txt.create_ctx(resp.text, optional=optional, n_workers=n_workers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c370bf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(get_llmstxt('https://llmstxt.org/llms.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2fc5a55",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def split_url(url):\n",
    "    \"Split `url` into base, path, and file name, normalising name to '/' if empty\"\n",
    "    parsed = urlparse(url.strip('/'))\n",
    "    base = f\"{parsed.scheme}://{parsed.netloc}\"\n",
    "    path,spl,fname = parsed.path.rpartition('/')\n",
    "    fname = spl+fname\n",
    "    if not path and not fname: path='/'\n",
    "    return base,path,fname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a92b74e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('https://claudette.answer.ai', '', '/path'),\n",
       " ('https://claudette.answer.ai', '/', ''),\n",
       " ('https://llmstxt.org', '/', ''),\n",
       " ('https://llmstxt.org', '/', '')]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urls = ('https://claudette.answer.ai/path/', 'https://claudette.answer.ai/', 'https://llmstxt.org', 'https://llmstxt.org/')\n",
    "\n",
    "[split_url(o) for o in urls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5337c0a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def _tryget(url):\n",
    "    \"Return response from `url` if `status_code!=404`, otherwise `None`\"\n",
    "    res = get(url)\n",
    "    return None if res.status_code==404 else url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "189f5b24",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def find_docs(url):\n",
    "    \"If available, return LLM-friendly llms.txt context or markdown file location from `url`\"\n",
    "    base,path,fname = split_url(url)\n",
    "    url = (base+path+fname).strip('/')\n",
    "    if fname=='/llms.txt': return url\n",
    "    if Path(fname).suffix in('.md', '.txt', '.rst'): return _tryget(url)\n",
    "    if '.' in fname: return _tryget(url+'.md') or find_docs(url[:url.rfind('/')])\n",
    "    res = _tryget(url+'/llms.txt')\n",
    "    if res: return res\n",
    "    res = _tryget(url+'/index.md')\n",
    "    if res: return res\n",
    "    res = _tryget(url+'/index.html.md')\n",
    "    if res: return res\n",
    "    res = _tryget(url+'/index-commonmark.md')\n",
    "    if res: return res\n",
    "    parsed_url = urlparse(url)\n",
    "    if parsed_url.path == '/' or not parsed_url.path: return None\n",
    "    return find_docs(urljoin(url, '..'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d1722d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "fl_url = 'https://answerdotai.github.io/fastlite'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b226407",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://answerdotai.github.io/fastlite/llms.txt'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_docs(fl_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14344890",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://claudette.answer.ai/llms.txt\n",
      "https://claudette.answer.ai/llms.txt\n",
      "https://llmstxt.org/llms.txt\n",
      "https://llmstxt.org/llms.txt\n"
     ]
    }
   ],
   "source": [
    "for o in urls: print(find_docs(o))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "439546d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| eval: false\n",
    "suffixes = [\"/\", \"/tmp\", \"/tmp/tmp/\"]\n",
    "for suff in suffixes:\n",
    "    for o in urls:  test_eq(find_docs(o), find_docs(o+suff))\n",
    "\n",
    "test_eq(find_docs(\"https://github.com\"), \"https://github.com/llms.txt\")\n",
    "test_eq(find_docs(\"https://github.com/AnswerDotAI\"), \"https://github.com/llms.txt\")\n",
    "test_eq(find_docs(\"https://github.com/AnswerDotAI/\"), \"https://github.com/llms.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "771d1208",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def read_docs(url, optional=False, n_workers=None, rm_comments=True, rm_details=True):\n",
    "    \"If available, return LLM-friendly llms.txt context or markdown file response for `url`\"\n",
    "    url = find_docs(url)\n",
    "    if url.endswith('/llms.txt'): res = get_llmstxt(url, optional=optional, n_workers=n_workers)\n",
    "    else: res = get(url).text\n",
    "    return clean_md(res, rm_comments=rm_comments, rm_details=rm_details)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94ec4289",
   "metadata": {},
   "source": [
    "## Export -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e9ee5c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#|hide\n",
    "#|eval: false\n",
    "from nbdev.doclinks import nbdev_export\n",
    "nbdev_export()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c01784b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}