Scrape_Process_Website.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7ebb682e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping completed!\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import urlparse, urljoin\n",
    "from collections import deque\n",
    "import csv\n",
    "\n",
    "def is_valid(url):\n",
    "    parsed = urlparse(url)\n",
    "    return bool(parsed.netloc) and bool(parsed.scheme)\n",
    "\n",
    "def get_internal_links(url, soup):\n",
    "    internal_links = set()\n",
    "    domain_name = urlparse(url).scheme + \"://\" + urlparse(url).netloc\n",
    "    for a_tag in soup.find_all(\"a\"):\n",
    "        href = a_tag.attrs.get(\"href\")\n",
    "        if href == \"\" or href is None:\n",
    "            continue\n",
    "        href = urljoin(url, href)\n",
    "        parsed_href = urlparse(href)\n",
    "        href = parsed_href.scheme + \"://\" + parsed_href.netloc + parsed_href.path\n",
    "        if not is_valid(href):\n",
    "            continue\n",
    "        if href in internal_links:\n",
    "            continue\n",
    "        if domain_name not in href:\n",
    "            continue\n",
    "        internal_links.add(href)\n",
    "    return internal_links\n",
    "\n",
    "def download_page(url, folder):\n",
    "    if '/cy' in url:\n",
    "        return None\n",
    "    parsed_url = urlparse(url)\n",
    "    page_name = parsed_url.path\n",
    "\n",
    "    # Get the file extension\n",
    "    file_extension = os.path.splitext(page_name)[1]\n",
    "\n",
    "    # Skip .zip and .doc files\n",
    "    if file_extension in ['.zip', '.doc', '.docx', '.xlsx']:\n",
    "        return None\n",
    "    response = requests.get(url)\n",
    "    if response.status_code == 200:\n",
    "        page_name = urlparse(url).path\n",
    "        if page_name == \"\":\n",
    "            page_name = \"index.txt\"\n",
    "        else:\n",
    "            # Extract the file name from the URL, and if it doesn't have an extension, append .html\n",
    "            page_name = page_name.strip(\"/\").replace(\"/\", \"_\")\n",
    "            if '.' not in page_name:\n",
    "                page_name += \".txt\"\n",
    "        \n",
    "        page_path = os.path.join(folder, page_name)\n",
    "        \n",
    "        # If the file is an HTML file, parse it with BeautifulSoup and save it as a formatted string.\n",
    "        if page_name.endswith(\".txt\"):\n",
    "            soup = BeautifulSoup(response.text, \"html.parser\")\n",
    "            text = soup.get_text(separator=' ', strip=True)\n",
    "            print(f\"Downloaded and stripped {page_name}\")\n",
    "            try:\n",
    "                with open(page_path, \"w\", encoding=\"utf-8\") as f:\n",
    "                    f.write(text)\n",
    "            except OSError:\n",
    "                # If the filename is too long, truncate it and save again\n",
    "                short_page_path = os.path.join(folder, page_name[:255])\n",
    "                with open(short_page_path, \"w\", encoding=\"utf-8\") as f:\n",
    "                    f.write(text)\n",
    "            return soup\n",
    "        else:\n",
    "            # For non-HTML files, write the response content as bytes.\n",
    "            with open(page_path, \"wb\") as f:\n",
    "                f.write(response.content)\n",
    "    return None\n",
    "\n",
    "def scrape_website(start_url, folder):\n",
    "    os.makedirs(folder, exist_ok=True)\n",
    "    visited_links = set()\n",
    "    try:\n",
    "        with open('downloaded_pages.csv', 'r', newline='') as f:\n",
    "            reader = csv.reader(f)\n",
    "            for row in reader:\n",
    "                visited_links.add(row[0])\n",
    "    except FileNotFoundError:\n",
    "        pass\n",
    "    links_to_visit = deque()\n",
    "    links_to_visit.append(start_url)\n",
    "    while links_to_visit:\n",
    "        url = links_to_visit.popleft()\n",
    "        if url in visited_links:\n",
    "            continue\n",
    "        soup = download_page(url, folder)\n",
    "        if soup is not None:\n",
    "            visited_links.add(url)\n",
    "            internal_links = get_internal_links(url, soup)\n",
    "            for link in internal_links:\n",
    "                if link not in visited_links:\n",
    "                    links_to_visit.append(link)\n",
    "        if len(visited_links) % 50 == 0:\n",
    "            with open('downloaded_pages.csv', 'w', newline='') as f:\n",
    "                writer = csv.writer(f)\n",
    "                for link in visited_links:\n",
    "                    writer.writerow([link])\n",
    "    print(\"Scraping completed!\")\n",
    "\n",
    "# Replace with your URL and folder name\n",
    "scrape_website(\"https://www.electoralcommission.org.uk/sites/default/files/2023-10/Accredited%20Observers%202023-10-02\", \"downloaded_pages\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47b733bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "how-maintain-your-registered-political-party/removing-your-party-register"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1595f55f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare old and new sets of docs for discrepancies\n",
    "import os\n",
    "import csv\n",
    "from itertools import zip_longest\n",
    "\n",
    "# Define the directories\n",
    "new_dir = \"/Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages\"\n",
    "old_dir = \"/Users/cardigan/Documents/GitHub/old_electoral_commision_chatbot/docs\"\n",
    "\n",
    "# Get the list of file names in each directory, without extensions\n",
    "new_files = {os.path.splitext(f)[0] for f in os.listdir(new_dir)}\n",
    "old_files = {os.path.splitext(f)[0] for f in os.listdir(old_dir)}\n",
    "\n",
    "# Find the files that are in new_dir but not in old_dir, and vice versa\n",
    "in_new_not_old = new_files - old_files\n",
    "in_old_not_new = old_files - new_files\n",
    "\n",
    "# Write the results to a CSV file\n",
    "with open('file_comparison.csv', 'w', newline='') as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerow(['in new but not old', 'in old but not new'])\n",
    "    for new_file, old_file in zip_longest(in_new_not_old, in_old_not_new):\n",
    "        writer.writerow([new_file, old_file])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "815f8699",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Strip header and footer, store as txt\n",
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "from reportlab.lib.pagesizes import letter\n",
    "from reportlab.pdfgen import canvas\n",
    "\n",
    "def get_files(folder):\n",
    "    return [f for f in os.listdir(folder) if f.endswith('.html') or f.endswith('.txt')]\n",
    "\n",
    "def extract_text(file_path):\n",
    "    with open(file_path, 'r', encoding='utf-8') as f:\n",
    "        soup = BeautifulSoup(f, 'html.parser')\n",
    "    return soup.get_text(separator=' ', strip=True)\n",
    "\n",
    "def remove_similar_structure(texts):\n",
    "    if not texts:\n",
    "        return []\n",
    "    lines = [text.splitlines() for text in texts]\n",
    "    common_start = 0\n",
    "    common_end = 0\n",
    "\n",
    "    for i in range(min(len(lines[0]), min(len(text) for text in lines))):\n",
    "        if all(lines[0][i] == text[i] for text in lines):\n",
    "            common_start = i + 1\n",
    "        else:\n",
    "            break\n",
    "    \n",
    "    for i in range(min(len(lines[0]), min(len(text) for text in lines))):\n",
    "        if all(lines[0][-(i + 1)] == text[-(i + 1)] for text in lines):\n",
    "            common_end = i + 1\n",
    "        else:\n",
    "            break\n",
    "\n",
    "    return ['\\n'.join(text[common_start: len(text) - common_end]) for text in lines]\n",
    "\n",
    "def process_folder(folder):\n",
    "    # Define the strings to search for\n",
    "    start_string = \"Part of our role includes conducting research and gathering data, and publishing reports and making recommendations about elections that take place across the UK. Our reports and data on past elections and referendums Electoral registration research Public attitudes Modernising voting: flexible voting feasibility studies Pilot evaluations Electoral fraud data Referendum question research Political registration and regulation Political registration and regulation We are responsible for registering political parties, publishing political finance data, and taking action if we have reason to suspect political finance law has been broken. Political party registration Financial reporting Our enforcement work Imprints News and views News and views Get the latest news from us, and read media handbooks for elections. We also hold consultations, and publish our responses to consultations from others. Media centre Elections Act Our priorities for reforming elections Parliamentary briefings Key correspondence Our consultations Our responses to consultations Resources Resources Use our resources to share information about voting and democracy. Take a look at our resources for young people. Download lesson plans and activities for use in the classroom or youth groups. Share our assets and get the message out. Resources for young people Resources for Educators Democratic engagement resources Back\"\n",
    "    end_string1 = \"Footer Content\"\n",
    "    end_string2 = \"Was this article helpful?\"\n",
    "    \n",
    "    files = get_files(folder)\n",
    "\n",
    "    # Process each file\n",
    "    for file in files:\n",
    "        file_path = os.path.join(folder, file)\n",
    "        text = extract_text(file_path)\n",
    "        \n",
    "        # Find the start and end indices of the text to keep\n",
    "        start_index = text.find(start_string)\n",
    "        end_index1 = text.find(end_string1)\n",
    "        end_index2 = text.find(end_string2)\n",
    "        \n",
    "        # If the start string was found, remove everything before it\n",
    "        if start_index != -1:\n",
    "            text = text[start_index + len(start_string):]\n",
    "        \n",
    "        # If the end strings were found, remove everything after them\n",
    "        if end_index1 != -1:\n",
    "            text = text[:end_index1]\n",
    "        if end_index2 != -1:\n",
    "            text = text[:end_index2]\n",
    "        \n",
    "        # Save the processed text as a .txt file\n",
    "        txt_file = os.path.splitext(file)[0] + '.txt'\n",
    "        txt_path = os.path.join(folder, txt_file)\n",
    "        with open(txt_path, 'w', encoding='utf-8') as f:\n",
    "            f.write(text.strip())\n",
    "\n",
    "# Replace 'downloaded_pages' with the folder where your HTML pages are stored\n",
    "folder = 'downloaded_pages'\n",
    "process_folder(folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2abededf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Skipping line 33: expected 50 fields, saw 51\n",
      "Skipping line 35: expected 50 fields, saw 51\n",
      "Skipping line 57: expected 50 fields, saw 51\n",
      "Skipping line 59: expected 50 fields, saw 51\n",
      "Skipping line 107: expected 50 fields, saw 51\n",
      "Skipping line 171: expected 50 fields, saw 51\n",
      "Skipping line 177: expected 50 fields, saw 51\n",
      "Skipping line 179: expected 50 fields, saw 51\n",
      "Skipping line 189: expected 50 fields, saw 51\n",
      "Skipping line 237: expected 50 fields, saw 51\n",
      "Skipping line 249: expected 50 fields, saw 51\n",
      "Skipping line 251: expected 50 fields, saw 51\n",
      "Skipping line 257: expected 50 fields, saw 51\n",
      "Skipping line 377: expected 50 fields, saw 51\n",
      "Skipping line 387: expected 50 fields, saw 51\n",
      "Skipping line 389: expected 50 fields, saw 51\n",
      "Skipping line 681: expected 50 fields, saw 51\n",
      "Skipping line 683: expected 50 fields, saw 51\n",
      "Skipping line 713: expected 50 fields, saw 51\n",
      "Skipping line 717: expected 50 fields, saw 51\n",
      "Skipping line 719: expected 50 fields, saw 51\n",
      "Skipping line 747: expected 50 fields, saw 51\n",
      "Skipping line 761: expected 50 fields, saw 51\n",
      "Skipping line 809: expected 50 fields, saw 51\n",
      "Skipping line 855: expected 50 fields, saw 51\n",
      "Skipping line 929: expected 50 fields, saw 51\n",
      "Skipping line 931: expected 50 fields, saw 51\n",
      "Skipping line 951: expected 50 fields, saw 51\n",
      "Skipping line 953: expected 50 fields, saw 51\n",
      "Skipping line 985: expected 50 fields, saw 51\n",
      "Skipping line 1023: expected 50 fields, saw 52\n",
      "Skipping line 1191: expected 50 fields, saw 51\n",
      "Skipping line 1229: expected 50 fields, saw 51\n",
      "Skipping line 1235: expected 50 fields, saw 51\n",
      "Skipping line 1253: expected 50 fields, saw 51\n",
      "Skipping line 1331: expected 50 fields, saw 51\n",
      "Skipping line 1349: expected 50 fields, saw 51\n",
      "Skipping line 1437: expected 50 fields, saw 51\n",
      "Skipping line 1475: expected 50 fields, saw 51\n",
      "Skipping line 1477: expected 50 fields, saw 51\n",
      "Skipping line 1527: expected 50 fields, saw 51\n",
      "Skipping line 1529: expected 50 fields, saw 51\n",
      "Skipping line 1531: expected 50 fields, saw 51\n",
      "Skipping line 1539: expected 50 fields, saw 51\n",
      "Skipping line 1541: expected 50 fields, saw 51\n",
      "Skipping line 1543: expected 50 fields, saw 51\n",
      "Skipping line 1545: expected 50 fields, saw 51\n",
      "Skipping line 1547: expected 50 fields, saw 51\n",
      "Skipping line 1549: expected 50 fields, saw 51\n",
      "Skipping line 1551: expected 50 fields, saw 51\n",
      "Skipping line 1601: expected 50 fields, saw 51\n",
      "Skipping line 1643: expected 50 fields, saw 51\n",
      "Skipping line 1907: expected 50 fields, saw 51\n",
      "Skipping line 1909: expected 50 fields, saw 51\n",
      "Skipping line 1939: expected 50 fields, saw 51\n",
      "Skipping line 1943: expected 50 fields, saw 51\n",
      "Skipping line 1945: expected 50 fields, saw 51\n",
      "Skipping line 1973: expected 50 fields, saw 51\n",
      "Skipping line 1987: expected 50 fields, saw 51\n",
      "Skipping line 2053: expected 50 fields, saw 51\n",
      "Skipping line 2055: expected 50 fields, saw 51\n",
      "Skipping line 2097: expected 50 fields, saw 51\n",
      "Skipping line 2167: expected 50 fields, saw 51\n",
      "Skipping line 2169: expected 50 fields, saw 51\n",
      "Skipping line 2205: expected 50 fields, saw 51\n",
      "Skipping line 2207: expected 50 fields, saw 51\n",
      "Skipping line 2209: expected 50 fields, saw 51\n",
      "Skipping line 2211: expected 50 fields, saw 51\n",
      "Skipping line 2215: expected 50 fields, saw 51\n",
      "Skipping line 2273: expected 50 fields, saw 52\n",
      "Skipping line 2475: expected 50 fields, saw 51\n",
      "Skipping line 2493: expected 50 fields, saw 51\n",
      "Skipping line 2511: expected 50 fields, saw 51\n",
      "Skipping line 2535: expected 50 fields, saw 51\n",
      "Skipping line 2589: expected 50 fields, saw 51\n",
      "Skipping line 2627: expected 50 fields, saw 51\n",
      "Skipping line 2713: expected 50 fields, saw 51\n",
      "Skipping line 2729: expected 50 fields, saw 51\n",
      "Skipping line 2751: expected 50 fields, saw 51\n",
      "Skipping line 2801: expected 50 fields, saw 51\n",
      "Skipping line 2803: expected 50 fields, saw 51\n",
      "Skipping line 2805: expected 50 fields, saw 51\n",
      "Skipping line 2807: expected 50 fields, saw 51\n",
      "Skipping line 2815: expected 50 fields, saw 51\n",
      "Skipping line 2817: expected 50 fields, saw 51\n",
      "Skipping line 2819: expected 50 fields, saw 51\n",
      "Skipping line 2821: expected 50 fields, saw 51\n",
      "Skipping line 2823: expected 50 fields, saw 51\n",
      "Skipping line 2827: expected 50 fields, saw 51\n",
      "Skipping line 2875: expected 50 fields, saw 51\n",
      "Skipping line 2909: expected 50 fields, saw 51\n",
      "Skipping line 2981: expected 50 fields, saw 51\n",
      "Skipping line 2983: expected 50 fields, saw 51\n",
      "Skipping line 3045: expected 50 fields, saw 51\n",
      "Skipping line 3157: expected 50 fields, saw 51\n",
      "Skipping line 3159: expected 50 fields, saw 51\n",
      "Skipping line 3163: expected 50 fields, saw 52\n",
      "Skipping line 3173: expected 50 fields, saw 51\n",
      "Skipping line 3227: expected 50 fields, saw 51\n",
      "Skipping line 3265: expected 50 fields, saw 52\n",
      "Skipping line 3299: expected 50 fields, saw 51\n",
      "Skipping line 3305: expected 50 fields, saw 51\n",
      "Skipping line 3307: expected 50 fields, saw 51\n",
      "Skipping line 3311: expected 50 fields, saw 51\n",
      "Skipping line 3317: expected 50 fields, saw 51\n",
      "Skipping line 3325: expected 50 fields, saw 51\n",
      "Skipping line 3327: expected 50 fields, saw 51\n",
      "Skipping line 3409: expected 50 fields, saw 51\n",
      "Skipping line 3465: expected 50 fields, saw 51\n",
      "Skipping line 3491: expected 50 fields, saw 52\n",
      "Skipping line 3493: expected 50 fields, saw 52\n",
      "Skipping line 3495: expected 50 fields, saw 52\n",
      "Skipping line 3497: expected 50 fields, saw 52\n",
      "Skipping line 3501: expected 50 fields, saw 51\n",
      "Skipping line 3509: expected 50 fields, saw 51\n",
      "Skipping line 3511: expected 50 fields, saw 51\n",
      "Skipping line 3519: expected 50 fields, saw 51\n",
      "Skipping line 3521: expected 50 fields, saw 51\n",
      "Skipping line 3619: expected 50 fields, saw 51\n",
      "Skipping line 3621: expected 50 fields, saw 51\n",
      "Skipping line 3631: expected 50 fields, saw 51\n",
      "Skipping line 3637: expected 50 fields, saw 51\n",
      "Skipping line 3639: expected 50 fields, saw 51\n",
      "Skipping line 3659: expected 50 fields, saw 51\n",
      "Skipping line 3677: expected 50 fields, saw 51\n",
      "Skipping line 3679: expected 50 fields, saw 51\n",
      "Skipping line 3685: expected 50 fields, saw 51\n",
      "Skipping line 3695: expected 50 fields, saw 51\n",
      "Skipping line 3809: expected 50 fields, saw 51\n",
      "Skipping line 3811: expected 50 fields, saw 51\n",
      "Skipping line 3813: expected 50 fields, saw 51\n",
      "Skipping line 4103: expected 50 fields, saw 51\n",
      "Skipping line 4125: expected 50 fields, saw 51\n",
      "Skipping line 4139: expected 50 fields, saw 51\n",
      "Skipping line 4157: expected 50 fields, saw 51\n",
      "Skipping line 4161: expected 50 fields, saw 51\n",
      "Skipping line 4191: expected 50 fields, saw 51\n",
      "Skipping line 4205: expected 50 fields, saw 51\n",
      "Skipping line 4247: expected 50 fields, saw 51\n",
      "Skipping line 4249: expected 50 fields, saw 51\n",
      "Skipping line 4295: expected 50 fields, saw 51\n",
      "Skipping line 4367: expected 50 fields, saw 51\n",
      "Skipping line 4369: expected 50 fields, saw 51\n",
      "Skipping line 4405: expected 50 fields, saw 51\n",
      "Skipping line 4407: expected 50 fields, saw 51\n",
      "Skipping line 4409: expected 50 fields, saw 51\n",
      "Skipping line 4411: expected 50 fields, saw 51\n",
      "Skipping line 4415: expected 50 fields, saw 51\n",
      "Skipping line 4475: expected 50 fields, saw 52\n",
      "Skipping line 4663: expected 50 fields, saw 51\n",
      "Skipping line 4681: expected 50 fields, saw 51\n",
      "Skipping line 4699: expected 50 fields, saw 51\n",
      "Skipping line 4707: expected 50 fields, saw 51\n",
      "Skipping line 4801: expected 50 fields, saw 51\n",
      "Skipping line 4887: expected 50 fields, saw 51\n",
      "Skipping line 4923: expected 50 fields, saw 51\n",
      "Skipping line 4925: expected 50 fields, saw 51\n",
      "Skipping line 4979: expected 50 fields, saw 51\n",
      "Skipping line 4991: expected 50 fields, saw 51\n",
      "Skipping line 4993: expected 50 fields, saw 51\n",
      "Skipping line 4995: expected 50 fields, saw 51\n",
      "Skipping line 4997: expected 50 fields, saw 51\n",
      "Skipping line 4999: expected 50 fields, saw 51\n",
      "Skipping line 5001: expected 50 fields, saw 51\n",
      "Skipping line 5011: expected 50 fields, saw 51\n",
      "Skipping line 5013: expected 50 fields, saw 51\n",
      "Skipping line 5023: expected 50 fields, saw 51\n",
      "Skipping line 5051: expected 50 fields, saw 51\n",
      "Skipping line 5249: expected 50 fields, saw 51\n",
      "Skipping line 5343: expected 50 fields, saw 51\n",
      "Skipping line 5349: expected 50 fields, saw 52\n",
      "Skipping line 5393: expected 50 fields, saw 51\n",
      "Skipping line 5415: expected 50 fields, saw 51\n",
      "Skipping line 5427: expected 50 fields, saw 51\n",
      "Skipping line 5741: expected 50 fields, saw 51\n",
      "Skipping line 5743: expected 50 fields, saw 51\n",
      "Skipping line 5773: expected 50 fields, saw 51\n",
      "Skipping line 5777: expected 50 fields, saw 51\n",
      "Skipping line 5779: expected 50 fields, saw 51\n",
      "Skipping line 5807: expected 50 fields, saw 51\n",
      "Skipping line 5821: expected 50 fields, saw 51\n",
      "Skipping line 5865: expected 50 fields, saw 51\n",
      "Skipping line 5887: expected 50 fields, saw 51\n",
      "Skipping line 5931: expected 50 fields, saw 51\n",
      "Skipping line 6001: expected 50 fields, saw 51\n",
      "Skipping line 6003: expected 50 fields, saw 51\n",
      "Skipping line 6021: expected 50 fields, saw 51\n",
      "Skipping line 6039: expected 50 fields, saw 51\n",
      "Skipping line 6041: expected 50 fields, saw 51\n",
      "Skipping line 6043: expected 50 fields, saw 51\n",
      "Skipping line 6047: expected 50 fields, saw 51\n",
      "Skipping line 6103: expected 50 fields, saw 52\n",
      "Skipping line 6305: expected 50 fields, saw 51\n",
      "Skipping line 6323: expected 50 fields, saw 51\n",
      "Skipping line 6341: expected 50 fields, saw 51\n",
      "Skipping line 6365: expected 50 fields, saw 51\n",
      "Skipping line 6419: expected 50 fields, saw 51\n",
      "Skipping line 6457: expected 50 fields, saw 51\n",
      "Skipping line 6543: expected 50 fields, saw 51\n",
      "Skipping line 6559: expected 50 fields, saw 51\n",
      "Skipping line 6581: expected 50 fields, saw 51\n",
      "Skipping line 6631: expected 50 fields, saw 51\n",
      "Skipping line 6633: expected 50 fields, saw 51\n",
      "Skipping line 6635: expected 50 fields, saw 51\n",
      "Skipping line 6637: expected 50 fields, saw 51\n",
      "Skipping line 6645: expected 50 fields, saw 51\n",
      "Skipping line 6647: expected 50 fields, saw 51\n",
      "Skipping line 6649: expected 50 fields, saw 51\n",
      "Skipping line 6651: expected 50 fields, saw 51\n",
      "Skipping line 6653: expected 50 fields, saw 51\n",
      "Skipping line 6657: expected 50 fields, saw 51\n",
      "Skipping line 6705: expected 50 fields, saw 51\n",
      "Skipping line 6739: expected 50 fields, saw 51\n",
      "Skipping line 6791: expected 50 fields, saw 51\n",
      "Skipping line 6793: expected 50 fields, saw 51\n",
      "Skipping line 6811: expected 50 fields, saw 51\n",
      "Skipping line 7081: expected 50 fields, saw 51\n",
      "Skipping line 7083: expected 50 fields, saw 51\n",
      "Skipping line 7117: expected 50 fields, saw 51\n",
      "Skipping line 7121: expected 50 fields, saw 51\n",
      "Skipping line 7123: expected 50 fields, saw 51\n",
      "Skipping line 7153: expected 50 fields, saw 51\n",
      "Skipping line 7217: expected 50 fields, saw 51\n",
      "Skipping line 7219: expected 50 fields, saw 51\n",
      "Skipping line 7307: expected 50 fields, saw 51\n",
      "Skipping line 7329: expected 50 fields, saw 51\n",
      "Skipping line 7345: expected 50 fields, saw 51\n",
      "Skipping line 7347: expected 50 fields, saw 51\n",
      "Skipping line 7349: expected 50 fields, saw 51\n",
      "Skipping line 7351: expected 50 fields, saw 51\n",
      "Skipping line 7359: expected 50 fields, saw 51\n",
      "Skipping line 7563: expected 50 fields, saw 51\n",
      "Skipping line 7589: expected 50 fields, saw 51\n",
      "Skipping line 7791: expected 50 fields, saw 51\n",
      "Skipping line 7793: expected 50 fields, saw 51\n",
      "Skipping line 7847: expected 50 fields, saw 51\n",
      "Skipping line 7849: expected 50 fields, saw 51\n",
      "Skipping line 7851: expected 50 fields, saw 51\n",
      "Skipping line 7853: expected 50 fields, saw 51\n",
      "Skipping line 7855: expected 50 fields, saw 51\n",
      "Skipping line 7863: expected 50 fields, saw 51\n",
      "Skipping line 7865: expected 50 fields, saw 51\n",
      "Skipping line 7867: expected 50 fields, saw 51\n",
      "Skipping line 7869: expected 50 fields, saw 51\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import io\n",
    "\n",
    "def get_csv_files(folder):\n",
    "    return [f for f in os.listdir(folder) if f.endswith('.csv')]\n",
    "\n",
    "folder = 'downloaded_pages'\n",
    "files = get_csv_files(folder)\n",
    "\n",
    "# Process each file\n",
    "for file in files:\n",
    "    file_path = os.path.join(folder, file)\n",
    "    \n",
    "    # Get the file size in bytes\n",
    "    file_size_bytes = os.path.getsize(file_path)\n",
    "    \n",
    "    # Convert the file size to MB\n",
    "    file_size_mb = file_size_bytes / (1024 * 1024)\n",
    "    \n",
    "    # Only process the file if it's under 3MB\n",
    "    if file_size_mb <= 3:\n",
    "        try:\n",
    "            # Read the file in binary mode, decode each line and handle errors\n",
    "            with open(file_path, 'rb') as f:\n",
    "                lines = f.readlines()\n",
    "            decoded_lines = [line.decode('utf-8', errors='ignore') for line in lines]\n",
    "            cleaned_data = io.StringIO('\\n'.join(decoded_lines))\n",
    "\n",
    "            # Read the cleaned data using pandas\n",
    "            df = pd.read_csv(cleaned_data, on_bad_lines = 'warn')\n",
    "\n",
    "            # Convert the DataFrame to Markdown format\n",
    "            markdown_text = df.to_markdown()\n",
    "\n",
    "            # Write to a text file\n",
    "            txt_file = os.path.splitext(file)[0] + '.txt'\n",
    "            txt_path = os.path.join(folder, txt_file)\n",
    "            with open(txt_path, 'w', encoding='utf-8') as f:\n",
    "                f.write(markdown_text.strip())\n",
    "        except pd.errors.ParserError as e:\n",
    "            print(f\"Failed to parse {file}: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d85182c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import textwrap\n",
    "\n",
    "# def save_text_as_pdf(file_path, text):\n",
    "#     c = canvas.Canvas(file_path, pagesize=letter)\n",
    "#     width, height = letter\n",
    "#     margin = 50  # Margin from each side\n",
    "#     font_size = 12\n",
    "#     line_spacing = 14\n",
    "#     c.setFont(\"Helvetica\", font_size)\n",
    "    \n",
    "#     # Ensure width is an integer\n",
    "#     wrapper = textwrap.TextWrapper(width=int((width - 2 * margin) // 6))  \n",
    "#     lines = []\n",
    "#     for line in text.splitlines():\n",
    "#         lines.extend(wrapper.wrap(line))\n",
    "    \n",
    "#     y_position = height - margin  \n",
    "#     for line in lines:\n",
    "#         if y_position < margin:  \n",
    "#             c.showPage()\n",
    "#             y_position = height - margin\n",
    "#         c.drawString(margin, y_position, line)\n",
    "#         y_position -= line_spacing\n",
    "    \n",
    "#     c.save()\n",
    "\n",
    "# def txt_to_pdf():\n",
    "#     # Save processed text as PDF\n",
    "#     for file, text in zip(html_files, processed_texts):\n",
    "#         # Creating the file name\n",
    "#         file_name = f\"{os.path.splitext(file)[0]}_processed.pdf\"\n",
    "        \n",
    "#         # Ensure the file name is not too long\n",
    "#         max_length = 255  # typical maximum filename length\n",
    "#         if len(file_name) > max_length:\n",
    "#             # Truncate the file name to the maximum length\n",
    "#             file_name = file_name[:max_length]\n",
    "        \n",
    "#         pdf_file = os.path.join(stripped_folder, file_name)\n",
    "#         save_text_as_pdf(pdf_file, text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c162f25c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_PCC%20Part%202b%20-%20Standing%20as%20a%20party%20candidate%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_UKPGE%20Part%202a%20Standing%20as%20an%20independent%20candidate%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Part%201%20Can%20you%20stand%20for%20election%20Community%20LGW%20W_1.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Register%20request%20form%20generic%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-06_Templed%20Datganiad%20i%27r%20Wasg%20i%20Gymru%20-%20Canfasiad%20blynyddol%202023.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Notice%20of%20election%20-%20region%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_PARO%20guidance%20-%20Delivery%20of%20key%20processes%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-09_The%20December%202022%20electoral%20registers%20in%20Wales.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-12_PCC%20Withdrawal%20form%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Declaration%20for%20the%20companion%20of%20a%20disabled%20voter%20LGW%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Nomination%20pack%20-%20region%20party%20list%20W.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-06_Data%20Protection%20resource%20for%20ERO%20and%20RO%20W.DOCX has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-04_Senedd%20PCC%20FAQs%20for%20frontline%20staff%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_2022%20Locals%20Wales%20Part%203%20C%26A_0.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_PCC%20Part%204%20-%20The%20campaign%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Withdrawal%20form%20-%20region%20-%20candidates%20included%20on%20a%20party%20list%20-%20for%20use%20either%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Nomination%20checklist%20regional%20W.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-12_UKPGE%20Part%206%20After%20the%20declaration%20of%20results%20-%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Counting%20agent%20appointment%20form%20generic%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Guidance%20on%20the%20retention%20and%20inspection%20of%20election%20documents%20LGW%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-12_PCC%20Withdrawal%20form.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Part%201%20-%20Can%20you%20stand%20for%20election%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_FAQs%20for%20frontline%20staff%20LGW%20W_1.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Supporting%20candidates%20and%20agents%20-%20May%202021%20polls%20W_0.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_PCC%20Part%202a%20-%20Standing%20as%20an%20independent%20candidate%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Senedd%20Part%204%20-%20The%20campaign%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_CCS207_CCS0221019662-001_2021-22%20VOA%20Web%20Accessible.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_List%20of%20EU%20member%20states%20generic%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_SP%20Withdrawal%20form%20region%20candidates%20included%20on%20a%20party%20list%20for%20use%20either%20by%20the%20partys%20NO.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-01_PCC%20timetable%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-06_NPC%20imprints%20England%20Wales%20Northern%20Ireland.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Polling%20station%20considerations%20for%202021%20polls%20-%20W_0.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_Parliamentary%20By%20Election%20Part%202a%20Standing%20as%20an%20independent%20candidate%20-%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_Close%20of%20poll%20packing%20instructions%20Wales%20holding%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Planning%20considerations%20for%202021%20polls%20-%20W_0.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Dummy%20W_6.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Declaration%20for%20the%20companion%20of%20a%20disabled%20voter%20Senedd%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Withdrawal%20form%20-%20constituency.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_Parliamentary%20By-Election%20-%20Part%204-The%20campaign%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20PCC%20Timetable%20W_0.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Categories%20of%20rejection%20and%20criteria%20for%20categorising%20rejections%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-02_LGW%20MAY%20RO%20Part%20C%20Administering%20the%20poll%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Part%205%20Your%20right%20to%20attend%20key%20electoral%20events%20LGW%20-%20W_0.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-04_Electoral%20Commission%20Regulations%20-%20Registered%20Parties%27%20Quarterly%20and%20Weekly%20Donation%20Reports%20%28Additional%20Information%29%20Regulations%202007.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-04_Educators%27%20notes%20-%20Wales%20%28Cymraeg%29%202023.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Proof%20checking%20factsheet%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-06_PCC%20Nomination%20pack%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Withdrawal%20form%20Communities%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-12_Part%206%20After%20the%20declaration%20of%20results%20LGW%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20PCC%20Secrecy%20-%20postal%20voting%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-12_Tellers%20dos%20and%20donts%20generic%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_UKPGE%20Part%202b%20Standing%20as%20a%20party%20candidate%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-06_PCC%20PARO%20Nomination%20checklist%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_CCS207_CCS0221019902-001_2020-21%20EC%20Supplementary%20Estimate%20A4%20Web%20Accessible.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Notice%20of%20election%20Wales%20LGW%20WB_0.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Polling%20station%20scenarios%20LGW%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_LGW%20dealing%20with%20doubtful%20ballot%20papers%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-03_Joint%20Guidance%20for%20Candidates%20in%20Elections%20-%20When%20it%20goes%20too%20far.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Contract%20development%20checklist%20RO%20W.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Dummy%20W_4.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Nominations%20pack%20updated%20-%20Community%20Council%20W.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-02_Senedd%20PCC%20Part%20C%20-%20Administering%20the%20poll%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_UKPGE%20Overview%20-%20W_0.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_UKPGE%20election%20timetable%20generic%20EA22%20holding%20W_0.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_MAY%20Withdrawal%20form_0.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_SLG%20Withdrawal%20Form.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-03_Candidate%20Webinar%20April%20%202023.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Nomination%20pack%20-%20constituency%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_LGW%20MAY%20RO%20Part%20E%20Verifying%20and%20counting%20the%20votes%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20doubtful%20ballot%20paper%20placemat%20%20constituency%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_LGW%20MAY%20RO%20Part%20F%20Post%20election%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Notice%20of%20election%20-%20constituency%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_PCC%20Doubtful%20ballot%20paper%20booklet%20W.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_FAQs%20for%20postal%20vote%20rejection%20notices%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Part%202b%20-%20Standing%20at%20the%20regional%20election%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Part%202a%20Standing%20as%20an%20independent%20candidate%20LGW%20W_0.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20PCC%20Secrecy%20-%20postal%20voting%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20PCC%20Scenarios%20for%20polling%20station%20staff%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Access%20supply%20and%20inspection%20checklist%20generic%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-08_Candidates%20imprints%20factsheet%20UKP%20England%20Wales%20Northern%20Ireland.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Polling%20station%20quiz%20LGW%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-03_Quality%20Assurance%20Guidance%20for%20ROs%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_SP%20Withdrawal%20form%20-%20region%20-%20withdrawing%20a%20full%20party%20list.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Withdrawal%20form%20-%20region%20-%20individual%20candidates%20not%20on%20a%20regional%20party%20list.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Part%202b%20Standing%20as%20a%20party%20candidate%20LGW%20W_0.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Verification%20and%20Count%20considerations%20for%202021%20polls%20-%20W_1.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_Ballot%20paper%20account%20exercise%20generic%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-03_Tellers%20dos%20and%20donts%20generic%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-12_Senedd%20Part%206%20-%20After%20the%20declaration%20of%20the%20results%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_Ballot%20paper%20account%20Wales%20generic%20WB.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20PCC%20Secrecy%20-%20the%20poll%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Absent%20voters%20list%20request%20form%20generic%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Part%204%20The%20campaign%20LGW%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-04_Verification%20and%20count%20toolkit%20Generic%20LGE%20and%20%20LGW%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_SP%20Withdrawal%20form%20-%20constituency.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Polling%20agent%20appointment%20form%20generic%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-07_Part%201%20Can%20you%20stand%20for%20election%20LGW%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_SP%20Withdrawal%20form%20region%20individual%20candidates%20not%20on%20a%20regional%20party%20list.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_LGW%20MAY%20RO%20Part%20D%20Absent%20voting%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Nominations%20considerations%20for%202021%20polls%20-%20W_0.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_PCC%20Doubtful%20ballot%20paper%20placemat%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Part%202a%20Standing%20as%20an%20independent%20candidate%20Community%20Council%20W_0.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Part%205%20Your%20right%20to%20attend%20key%20electoral%20events%20Community%20W_0.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Part%202a%20-%20Standing%20at%20the%20constituency%20election%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Withdrawal%20form%20-%20region%20-%20withdrawing%20a%20full%20party%20list%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-10_Senedd%20PCC%20Part%20A%20-%20Role%20and%20responsibilities%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20doubtful%20ballot%20paper%20placemat%20region%20rejected%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_UKPGE%20Part%201%20Can%20you%20stand%20for%20election%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Template%20postal%20vote%20rejection%20notices%20W_0.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-01_PCC%20-%20Overview%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd___PCC%202021%20Quick_Guide%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-04_Senedd%20Notice%20of%20poll%20and%20situation%20of%20polling%20stations%20-%20constituency%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_PCC%20Part%201%20-%20Can%20you%20stand%20for%20election%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20doubtful%20ballot%20paper%20placemat%20region%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Withdrawal%20form%20-%20region%20-%20individual%20candidates%20not%20on%20a%20regional%20party%20list%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-03_Tellers%20guidance%20generic%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-12_Part%206%20After%20the%20declaration%20of%20results%20Community%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_Parliamentary%20By-Election%20-%20Part%206-After%20the%20declaration%20of%20results%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_Parliamentary%20By%20Election%20Overview%20-%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-01_PCC%20Election%20address%20factsheet%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Community%20council%20elections%20in%20Wales%20-%20combined%20spending%20form%20declaration%20and%20notes%20-%20updated%20Jan%2022.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-03_Notice%20of%20poll%20LGW%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-10_Senedd%20PCC%20Part%20B%20-%20Planning%20and%20organisation%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Withdrawal%20form%20-%20constituency%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_UKPGE%20Part%204%20The%20campaign%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20PCC%20Secrecy%20-%20the%20count%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_LGW%20Nominations%20pack%20updated%20%20W.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Overview%20Community%20LGW%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-10_Senedd%20PCC%20Part%20E%20-%20Verifying%20and%20counting%20the%20votes%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_PCC%20Guidance%20on%20the%20retention%20and%20inspection%20of%20election%20documents%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Nomination%20pack%20-%20region%20individual%20W.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Part%202b%20Standing%20as%20a%20party%20candidate%20Community%20Council%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Withdrawal%20form%20-%20region%20-%20withdrawing%20a%20full%20party%20list.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_Parliamentary%20By%20Election%20Part%202b%20Standing%20as%20a%20party%20candidate%20-%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_Parliamentary%20By%20Election%20Part%201%20Can%20you%20stand%20for%20election%20-%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_Personation%20advice%20for%20Presiding%20Officers%20Wales%20generic%20WB.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Overview%20LGW%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20PCC%20Part%20D%20-%20Absent%20Voting%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Nomination%20checklist%20constituency%20W.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_PCC%20Part%206%20-%20After%20the%20declaration%20of%20the%20results%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_List%20of%20Commonwealth%20countries%20generic%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-09_Key%20considerations%20for%20Tranche%202%20of%20the%20Elections%20Act%202023%20W%20Translation.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_Checklist%20for%20polling%20station%20inspectors%20generic%20EA22%20holding%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_PCC%20Part%205%20-%20Your%20right%20to%20attend%20key%20electoral%20events%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-09_UKPGE%20Part%205%20Your%20right%20to%20attend%20key%20electoral%20events%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Withdrawal%20form%20-%20region%20-%20candidates%20included%20on%20a%20party%20list%20-%20for%20use%20either.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_Parliamentary%20By-Election%20-%20Part%205-Your%20right%20to%20attend%20key%20electoral%20events%20holding%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_PCC%20Doubtful%20ballot%20paper%20placemat%20W_0.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-10_Senedd%20Part%205%20-%20Your%20right%20to%20attend%20key%20electoral%20events%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_LGW%20Template%20bilingual%20count%20scripts%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Part%204%20The%20campaign%20Community%20LGW%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Design%20criteria%20forms%20and%20notices%20generic%20W.doc has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-03_Code%20of%20conduct%20for%20campaigners%202015%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20PCC%20Part%20F%20-%20After%20the%20declaration%20of%20the%20results%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-06_Absent%20voting%20considerations%20for%202021%20polls%20-%20W.docx has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20doubtful%20ballot%20paper%20booklet%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-01_ER%20letter%20for%20Wales.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-10_Senedd%20-%20Overview%20W.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20notice%20of%20poll%20and%20situation%20of%20polling%20stations%20%20-%20region%20W_0.doc has been deleted\n"
     ]
    }
   ],
   "source": [
    "###Delete pages with specific substring in filename\n",
    "\n",
    "import os\n",
    "import glob\n",
    "\n",
    "directory = \"/Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages\"  # replace with the path to your directory\n",
    "substring = \"20W\"  # replace with the substring to search for\n",
    "\n",
    "# Use glob to list all files in the directory that contain the substring\n",
    "for file_path in glob.glob(os.path.join(directory, f'*{substring}*')):\n",
    "    try:\n",
    "        os.remove(file_path)\n",
    "        print(f\"File {file_path} has been deleted\")\n",
    "    except Exception as e:\n",
    "        print(f\"Error occurred: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f8b067f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A4-polling-station-poster-Urdu.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Making-Your-Mark-Example-Postal-Voting-Quick-Start-Guide-First-Past-The-Post-Election-GB-English-A4.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0015_107511_form-co1-ri.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_Your-vote-is-yours-alone-leaflet.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A4-postal-voting-poster-Bengali.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_C56013-EC-2pp_210x148.5.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20doubtful%20ballot%20paper%20placemat%20constituency.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0014_214232_Parliamentary-By-Election-Part-5-Your-right-to-attend-key-electoral-events.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Making-Your-Mark-Example-Postal-Voting-Quick-Start-Guide-First-Past-The-Post-Election-Northern-Ireland-English-A4.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_C56013-EC-2pp_210x148.5_Crops.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0020_214229_Parliamentary-By-Election-Part-2a-Standing-as-an-independent-candidate.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0020_214409_SLG-Part-B-Planning-and-organisation.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_electoral_commission_pdf_file_0018_13509_form-rd1b-rd.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Making-Your-Mark-Example-Booth-Notice-Bilingual-Colour-A2.pdf: cannot open empty document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_Your-vote-is-yours-alone-leaflet-with-print-marks.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0013_214411_SLG-Part-D-Absent-voting.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0013_140611_enote-rd1c.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Notional-Spending-Factsheet-2019.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Making-Your-Mark-Example-Postal-Voting-Quick-Start-Guide-First-Past-The-Post-Election-GB-Bilingual-A4.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0006_194586_Spending-for-EU-referendum-campaigners.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0012_214410_SLG-Part-C-Administering-the-poll.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A3-postal-voting-poster-Bengali.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A3-polling-station-poster-Punjabi.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A4-postal-voting-poster-UK.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Making-Your-Mark-Example-Postal-Voting-Statement-GB-English-A4.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Making-Your-Mark-Example-Postal-Voting-Quick-Start-Guide-Single-Transferable-Vote-Election-Northern-Ireland-English-A4.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0018_214227_Parliamentary-By-Election-Overview.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A4-polling-station-poster-UK.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0012_214230_Parliamentary-By-Election-Part-2b-Standing-as-a-party-candidate.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0014_214412_SLG-Part-E-Verifying-and-counting-the-votes.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A3-polling-station-poster-Urdu.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A3-polling-station-poster-UK.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Making-Your-Mark-Example-Postal-Voting-Quick-Start-Guide-Single-Transferable-Vote-Election-Scotland-English-A4.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Portfolio-3-docs-80-107.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0020_140627_enote-rd1c-ni.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0015_214413_SLG-Part-F-After-the-declaration-of-result.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A3-polling-station-poster-Bengali.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A3-postal-voting-poster-Punjabi.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A4-postal-voting-poster-Urdu.pdf has been deleted\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A4-postal-voting-poster-Punjabi.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_electoral_commission_pdf_file_0016_13507_form-rd1a-rd.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A4-polling-station-poster-Bengali.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0016_140614_enote-rd1d.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-09_A3-postal-voting-poster-UK.pdf has been deleted\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0015_214233_Parliamentary-By-Election-Part-6-After-the-declaration-of-results.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0014_107510_form-r1-ro.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0019_214408_SLG-Part-A-Returning-Officer-role-and-responsibilities.pdf: cannot open broken document\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0019_214228_Parliamentary-By-Election-Part-1-Can-you-stand-for-election.pdf: cannot open broken document\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/__data_assets_pdf_file_0013_214231_Parliamentary-By-Election-Part-4-The-campaign.pdf: cannot open broken document\n",
      "File /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Making-Your-Mark-Example-Postal-Voting-Statement-GB-Bilingual-A4.pdf has been deleted\n"
     ]
    }
   ],
   "source": [
    "###Delete posters using filesize to num pages ratio\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import fitz  # PyMuPDF\n",
    "\n",
    "directory = \"/Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages\"  # replace with the path to your directory\n",
    "max_size_per_page_kb = 500  # in KB\n",
    "\n",
    "# Use glob to list all PDF files in the directory\n",
    "for file_path in glob.glob(os.path.join(directory, '*.pdf')):\n",
    "    try:\n",
    "        # Get the file size in KB\n",
    "        file_size_kb = os.path.getsize(file_path) / 1024.0  # convert bytes to KB\n",
    "        \n",
    "        # Open the PDF file and get the number of pages\n",
    "        doc = fitz.open(file_path)\n",
    "        num_pages = doc.page_count\n",
    "        \n",
    "        # Calculate the size per page\n",
    "        size_per_page_kb = file_size_kb / num_pages\n",
    "        \n",
    "        # If the size per page is more than the allowed maximum, delete the file\n",
    "        if size_per_page_kb > max_size_per_page_kb:\n",
    "            os.remove(file_path)\n",
    "            print(f\"File {file_path} has been deleted\")\n",
    "        \n",
    "        # Close the document\n",
    "        doc.close()\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"Error occurred while processing {file_path}: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "063683d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Making-Your-Mark-Example-Booth-Notice-Bilingual-Colour-A2.pdf: cannot open empty document. Deleting the file.\n",
      "Error occurred while processing /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_pdf_file_Notional-Spending-Factsheet-2019.pdf: cannot open broken document. Deleting the file.\n"
     ]
    }
   ],
   "source": [
    "###Delete broken files\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import fitz  # PyMuPDF\n",
    "\n",
    "directory = \"/Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages\"  # replace with the path to your directory\n",
    "\n",
    "# Use glob to list all files in the directory\n",
    "for file_path in glob.glob(os.path.join(directory, '*')):\n",
    "    try:\n",
    "        # Try to open the document, if it is a PDF\n",
    "        if file_path.endswith('.pdf'):\n",
    "            doc = fitz.open(file_path)\n",
    "            doc.close()\n",
    "    except Exception as e:\n",
    "        print(f\"Error occurred while processing {file_path}: {e}. Deleting the file.\")\n",
    "        os.remove(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9d7c97aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-02_LGE%20election%20timetable%204%20May%202023_0.docx to sites_default_files_2023-02_LGE%20election%20timetable%204%20May%202023_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Staff%20Guide%20Plain%20Text%20English%20-%20Anonymous%20Voters.docx to sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Staff%20Guide%20Plain%20Text%20English%20-%20Anonymous%20Voters.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_List%20of%20Commonwealth%20countries%20generic.doc to sites_default_files_2022-01_List%20of%20Commonwealth%20countries%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Voter%20Booklet%20Plain%20Text%20English%20-%20Anonymous%20Voters.docx to sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Voter%20Booklet%20Plain%20Text%20English%20-%20Anonymous%20Voters.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_SP-Nomination-pack-region-party-list.doc to sites_default_files_2021-02_SP-Nomination-pack-region-party-list.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_SLG-Scheduled%20Election-timetable%202022.docx to sites_default_files_2022-01_SLG-Scheduled%20Election-timetable%202022.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_Senedd%20Nomination%20checklist%20constituency.docx to sites_default_files_2021-08_Senedd%20Nomination%20checklist%20constituency.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Verification%20and%20Count%20considerations%20for%202021%20polls%20-E%20version%20with%20March%20updates.docx to sites_default_files_2021-07_Verification%20and%20Count%20considerations%20for%202021%20polls%20-E%20version%20with%20March%20updates.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_PCC%20PARO%20Nomination%20checklist.doc to sites_default_files_2022-01_PCC%20PARO%20Nomination%20checklist.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Absent%20voters%20list%20request%20form%20generic.doc to sites_default_files_2022-01_Absent%20voters%20list%20request%20form%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Postal-voting-agent-appointment-form-generic.doc to sites_default_files_word_doc_Postal-voting-agent-appointment-form-generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_SP%20Nomination%20pack%20-%20constituency.doc to sites_default_files_2021-08_SP%20Nomination%20pack%20-%20constituency.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-03_Nomination%20pack%20parish%20council%20election_0.doc to sites_default_files_2023-03_Nomination%20pack%20parish%20council%20election_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_SP-Quiz-for-polling-station-staff.doc to sites_default_files_2021-02_SP-Quiz-for-polling-station-staff.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Checklist%20for%20polling%20station%20inspectors%20generic.doc to sites_default_files_2022-01_Checklist%20for%20polling%20station%20inspectors%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_FAQs%20for%20frontline%20staff%20LGW_1.docx to sites_default_files_2022-02_FAQs%20for%20frontline%20staff%20LGW_1.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Notice%20of%20poll%20and%20situation%20of%20polling%20stations%20-%20constituency.doc to sites_default_files_2021-02_Senedd%20Notice%20of%20poll%20and%20situation%20of%20polling%20stations%20-%20constituency.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Staff%20Guide%20Plain%20Text%20English%20-%20Generic.docx to sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Staff%20Guide%20Plain%20Text%20English%20-%20Generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_GLA%20FAQs%20for%20frontline%20staff_0.doc to sites_default_files_2021-03_GLA%20FAQs%20for%20frontline%20staff_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_LGE-PCC-Secrecy-the-count.doc to sites_default_files_word_doc_LGE-PCC-Secrecy-the-count.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_REDUCED%20SUBSCRIBERS%20PCC%20Nomination%20pack.doc to sites_default_files_2022-01_REDUCED%20SUBSCRIBERS%20PCC%20Nomination%20pack.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-03_SLG%20FAQs%20for%20frontline%20staff.doc to sites_default_files_2022-03_SLG%20FAQs%20for%20frontline%20staff.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-04_AccessibilitySurvey%20-%20guidance%20-%20easy%20read.docx to sites_default_files_2023-04_AccessibilitySurvey%20-%20guidance%20-%20easy%20read.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20PCC%20Secrecy%20-%20the%20poll.doc to sites_default_files_2021-02_Senedd%20PCC%20Secrecy%20-%20the%20poll.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20PCC%20Scenarios%20for%20polling%20station%20staff.doc to sites_default_files_2021-02_Senedd%20PCC%20Scenarios%20for%20polling%20station%20staff.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Nominations%20pack%20updated%20-%20Community%20Council.docx to sites_default_files_2023-07_Nominations%20pack%20updated%20-%20Community%20Council.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_Invitation-to-register-large-print-letter-GB.docx to sites_default_files_2021-08_Invitation-to-register-large-print-letter-GB.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_LG-Appointment-polling-agent-Final.doc to sites_default_files_word_doc_LG-Appointment-polling-agent-Final.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_GLA%20FAQs%20for%20frontline%20staff.doc to sites_default_files_2021-08_GLA%20FAQs%20for%20frontline%20staff.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Nomination%20pack%20-%20region%20party%20list.docx to sites_default_files_2021-03_Senedd%20Nomination%20pack%20-%20region%20party%20list.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_SLG%20Secrecy%20requirements%20_%20the%20poll.doc to sites_default_files_2022-01_SLG%20Secrecy%20requirements%20_%20the%20poll.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Nominations%20considerations%20for%202021%20polls%20-%20S.docx to sites_default_files_2021-07_Nominations%20considerations%20for%202021%20polls%20-%20S.pdf\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_LGW%20Nominations%20pack%20updated.docx to sites_default_files_2023-07_LGW%20Nominations%20pack%20updated.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-10_UKPGE-Election-timetable-%2012%20December_4.doc to sites_default_files_2019-10_UKPGE-Election-timetable-%2012%20December_4.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20PCC%20Secrecy%20-%20the%20count.doc to sites_default_files_2021-02_Senedd%20PCC%20Secrecy%20-%20the%20count.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Counting%20agent%20appointment%20form%20generic.doc to sites_default_files_2021-02_Counting%20agent%20appointment%20form%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_SLG%20Declaration%20for%20the%20companion%20of%20a%20voter%20with%20disabilities.docx to sites_default_files_2021-11_SLG%20Declaration%20for%20the%20companion%20of%20a%20voter%20with%20disabilities.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-02_Postal%20vote%20opening%20flowchart%20generic.docx to sites_default_files_2023-02_Postal%20vote%20opening%20flowchart%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_REDUCED%20SUBSCRIBERS%20PFCC%20Nomination%20pack.doc to sites_default_files_2022-01_REDUCED%20SUBSCRIBERS%20PFCC%20Nomination%20pack.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_EIA%20template%20-%20English.docx to sites_default_files_2021-11_EIA%20template%20-%20English.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Register%20request%20form%20generic.doc to sites_default_files_2021-02_Register%20request%20form%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Postal-vote-retrieval-flowchart-generic.doc to sites_default_files_word_doc_Postal-vote-retrieval-flowchart-generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_PCC-Secrecy-the-count.doc to sites_default_files_word_doc_PCC-Secrecy-the-count.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_SLG%20Polling%20station%20quiz.doc to sites_default_files_2021-11_SLG%20Polling%20station%20quiz.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_LGW%20scheduled%20election-timetable%20%202022.docx to sites_default_files_2022-02_LGW%20scheduled%20election-timetable%20%202022.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-04_AccessibilitySurvey%20-%20guidance%20-%20final.docx to sites_default_files_2023-04_AccessibilitySurvey%20-%20guidance%20-%20final.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Observer-access-restrictions-log-generic.doc to sites_default_files_word_doc_Observer-access-restrictions-log-generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_SP%20timetable_0.doc to sites_default_files_2021-03_SP%20timetable_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_SLG-Secrecy-requirements-count.doc to sites_default_files_word_doc_SLG-Secrecy-requirements-count.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_SP-Secrecy-the-count.docx to sites_default_files_word_doc_SP-Secrecy-the-count.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Polling station considerations for 2021 polls - E.docx to sites_default_files_2021-07_Polling station considerations for 2021 polls - E.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_SP-Secrecy-the-poll.doc to sites_default_files_word_doc_SP-Secrecy-the-poll.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_LGW%20Template%20bilingual%20count%20scripts.doc to sites_default_files_2022-01_LGW%20Template%20bilingual%20count%20scripts.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Polling%20station%20quiz%20LGW_0.doc to sites_default_files_2022-01_Polling%20station%20quiz%20LGW_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-05_UKPE%20By%20Election%20timetable%20generic_0.doc to sites_default_files_2022-05_UKPE%20By%20Election%20timetable%20generic_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_SP%20Declaration%20for%20the%20companion%20of%20a%20voter%20with%20disabilities.docx to sites_default_files_2021-07_SP%20Declaration%20for%20the%20companion%20of%20a%20voter%20with%20disabilities.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Withdrawal%20form%20Community.doc to sites_default_files_2022-01_Withdrawal%20form%20Community.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Template%20postal%20vote%20rejection%20notices.doc to sites_default_files_2021-02_Template%20postal%20vote%20rejection%20notices.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-07_Election-timetable-LGEW-Generic.doc to sites_default_files_2019-07_Election-timetable-LGEW-Generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Voter%20Booklet%20Plain%20Text%20English%20-%20Generic.docx to sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Voter%20Booklet%20Plain%20Text%20English%20-%20Generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-03_LGE%20reduced%20subscribers%20Nomination%20pack%20incl%20election%20agent%20notification%20form_1.docx to sites_default_files_2023-03_LGE%20reduced%20subscribers%20Nomination%20pack%20incl%20election%20agent%20notification%20form_1.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Nomination%20pack%20-%20region%20individual.docx to sites_default_files_2021-03_Senedd%20Nomination%20pack%20-%20region%20individual.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_EIA%20Screening%20-%20English.docx to sites_default_files_2021-11_EIA%20Screening%20-%20English.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Re-issuing-of-lost-and-spoilt-postal-votes-flowchart-generic.doc to sites_default_files_word_doc_Re-issuing-of-lost-and-spoilt-postal-votes-flowchart-generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Postal-vote-issuing-flowchart-generic.doc to sites_default_files_word_doc_Postal-vote-issuing-flowchart-generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Polling%20station%20considerations%20for%202021%20polls%20-%20E.docx to sites_default_files_2021-07_Polling%20station%20considerations%20for%202021%20polls%20-%20E.pdf\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Polling%20station%20scenarios%20LGW.doc to sites_default_files_2022-01_Polling%20station%20scenarios%20LGW.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20PCC%20FAQs%20for%20frontline%20staff.doc to sites_default_files_2021-03_Senedd%20PCC%20FAQs%20for%20frontline%20staff.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Planning%20considerations%20for%202021%20polls.docx to sites_default_files_2021-07_Planning%20considerations%20for%202021%20polls.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_Social_Media_Template_Posts.docx to sites_default_files_2021-11_Social_Media_Template_Posts.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_SLG%20Nomination%20packs%20and%20guidance%20for%20candidates%20and%20agents.doc to sites_default_files_2023-05_SLG%20Nomination%20packs%20and%20guidance%20for%20candidates%20and%20agents.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Notice%20of%20election%20-%20constituency.doc to sites_default_files_2021-03_Senedd%20Notice%20of%20election%20-%20constituency.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_SoA-Template-2018.doc to sites_default_files_word_doc_SoA-Template-2018.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_LGE-PCC-Secrecy-postal-voting.doc to sites_default_files_word_doc_LGE-PCC-Secrecy-postal-voting.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_doc-financial-rp.doc to sites_default_files_word_doc_doc-financial-rp.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-07_CAM%20Nomination%20pack%20incl%20election%20agent%20notification%20form%20-%20with%20PCC%20function.doc to sites_default_files_2022-07_CAM%20Nomination%20pack%20incl%20election%20agent%20notification%20form%20-%20with%20PCC%20function.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_PCC-Secrecy-postal-voting.doc to sites_default_files_word_doc_PCC-Secrecy-postal-voting.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-07_CAM%20Nomination%20pack%20incl%20election%20agent%20notification%20form%20-%20without%20PCC%20function.doc to sites_default_files_2022-07_CAM%20Nomination%20pack%20incl%20election%20agent%20notification%20form%20-%20without%20PCC%20function.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_PCC%20Notice%20of%20election.doc to sites_default_files_2021-03_PCC%20Notice%20of%20election.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_SP%20notice%20of%20poll%20and%20situation%20of%20polling%20stations%20%20-%20region.doc to sites_default_files_2021-02_SP%20notice%20of%20poll%20and%20situation%20of%20polling%20stations%20%20-%20region.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Planning%20considerations%20for%202021%20polls%20-%20S.docx to sites_default_files_2021-07_Planning%20considerations%20for%202021%20polls%20-%20S.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20PCC%20Timetable_1.doc to sites_default_files_2021-03_Senedd%20PCC%20Timetable_1.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-07_Log-to-record-details-of-postal-voters-who-claim-not-to-have-applied-for-a-postal-vote-generic.doc to sites_default_files_2019-07_Log-to-record-details-of-postal-voters-who-claim-not-to-have-applied-for-a-postal-vote-generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Polling-station-accessibility-checklist-generic.doc to sites_default_files_2022-01_Polling-station-accessibility-checklist-generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Nomination%20checklist%20for%20Returning%20Officers%20LGW%20-%20Principal%20Area.doc to sites_default_files_2022-02_Nomination%20checklist%20for%20Returning%20Officers%20LGW%20-%20Principal%20Area.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Secrecy-requirements-postal-voting-S66.doc to sites_default_files_word_doc_Secrecy-requirements-postal-voting-S66.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_GLA%20Secrecy%20-%20the%20poll.doc to sites_default_files_2021-08_GLA%20Secrecy%20-%20the%20poll.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Secrecy-requirements-the-count-MREF.doc to sites_default_files_word_doc_Secrecy-requirements-the-count-MREF.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Secrecy-requirements-the-poll-S66.doc to sites_default_files_word_doc_Secrecy-requirements-the-poll-S66.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-01_GLA%20timetable.doc to sites_default_files_2021-01_GLA%20timetable.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Nomination%20pack%20-%20constituency_2.docx to sites_default_files_2021-03_Senedd%20Nomination%20pack%20-%20constituency_2.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_SLG-Secrecy-requirements-postal-voting.doc to sites_default_files_word_doc_SLG-Secrecy-requirements-postal-voting.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_SLG%20Notice%20of%20election.doc to sites_default_files_2021-11_SLG%20Notice%20of%20election.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Supporting%20candidates%20and%20agents%20-%20May%202021%20polls%20-%20E.docx to sites_default_files_2021-07_Supporting%20candidates%20and%20agents%20-%20May%202021%20polls%20-%20E.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Polling%20station%20considerations%20for%202021%20polls%20-%20S.docx to sites_default_files_2021-07_Polling%20station%20considerations%20for%202021%20polls%20-%20S.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Absent%20voting%20considerations%20for%202021%20polls%20-%20S.docx to sites_default_files_2021-07_Absent%20voting%20considerations%20for%202021%20polls%20-%20S.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_GLA%20Secrecy%20-%20the%20count.doc to sites_default_files_2021-08_GLA%20Secrecy%20-%20the%20count.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Notice%20of%20poll%20LGW.doc to sites_default_files_2022-01_Notice%20of%20poll%20LGW.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-06_Absent%20voting%20considerations%20for%202021%20polls%20-%20E.docx to sites_default_files_2021-06_Absent%20voting%20considerations%20for%202021%20polls%20-%20E.pdf\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_UKP-Nomination-pack-incl-election-agent-notification-form.doc to sites_default_files_word_doc_UKP-Nomination-pack-incl-election-agent-notification-form.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Secrecy-requirements-the-poll-MREF.doc to sites_default_files_word_doc_Secrecy-requirements-the-poll-MREF.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_SP%20Nomination%20checklist%20-%20constituency.doc to sites_default_files_2021-08_SP%20Nomination%20checklist%20-%20constituency.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Election%20timetable%20LGE%20Generic.docx to sites_default_files_2022-01_Election%20timetable%20LGE%20Generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-05_Data%20Protection%20resource%20for%20ERO%20and%20RO.docx to sites_default_files_2021-05_Data%20Protection%20resource%20for%20ERO%20and%20RO.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Secrecy-requirements-postal-voting-MREF.doc to sites_default_files_word_doc_Secrecy-requirements-postal-voting-MREF.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Close%20of%20poll%20packing%20instructions%20generic_0.doc to sites_default_files_2021-02_Close%20of%20poll%20packing%20instructions%20generic_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-12_Dummy%20E_3_26.doc to sites_default_files_2020-12_Dummy%20E_3_26.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-07_Secrecy-requirements-verification-and-count-S66%20%281%29.doc to sites_default_files_2019-07_Secrecy-requirements-verification-and-count-S66%20%281%29.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_PCC-Secrecy-the-poll.doc to sites_default_files_word_doc_PCC-Secrecy-the-poll.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_RO%20RRO%20contact%20details%20SP%20election%202021.docx to sites_default_files_2021-02_RO%20RRO%20contact%20details%20SP%20election%202021.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Polling%20agent%20appointment%20form%20generic.doc to sites_default_files_2021-02_Polling%20agent%20appointment%20form%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20Guidance%20on%20the%20retention%20and%20inspection%20of%20election%20documents.doc to sites_default_files_2021-02_Senedd%20Guidance%20on%20the%20retention%20and%20inspection%20of%20election%20documents.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_SP-Guidance-on-the-retention-and-inspection-of-election-documents.doc to sites_default_files_word_doc_SP-Guidance-on-the-retention-and-inspection-of-election-documents.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_SLG%20Nomination%20checklist%20for%20Returning%20Officers.doc to sites_default_files_2021-11_SLG%20Nomination%20checklist%20for%20Returning%20Officers.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Postal-voting-ballot-paper-account-generic.doc to sites_default_files_word_doc_Postal-voting-ballot-paper-account-generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_List%20of%20EU%20member%20states%20generic.doc to sites_default_files_2021-11_List%20of%20EU%20member%20states%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Contract%20development%20checklist%20RO.docx to sites_default_files_2022-01_Contract%20development%20checklist%20RO.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_List%20of%20EU%20member%20states%20generic.doc to sites_default_files_2022-01_List%20of%20EU%20member%20states%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_SP%20Notice%20of%20election%20-%20region_0.doc to sites_default_files_2021-02_SP%20Notice%20of%20election%20-%20region_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2020-11_PCC%20timetable.doc to sites_default_files_2020-11_PCC%20timetable.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_LGE-PCC-Secrecy-the-poll.doc to sites_default_files_word_doc_LGE-PCC-Secrecy-the-poll.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Access%20supply%20and%20inspection%20checklist%20generic.doc to sites_default_files_2022-01_Access%20supply%20and%20inspection%20checklist%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-07_Withdrawal-form-LGEW.doc to sites_default_files_2019-07_Withdrawal-form-LGEW.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-11_Re-issuing%20of%20lost%20and%20spoilt%20postal%20votes%20flowchart%20generic.doc to sites_default_files_2019-11_Re-issuing%20of%20lost%20and%20spoilt%20postal%20votes%20flowchart%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Voter%20Booklet%20Easy%20Read%20English%20-%20Generic_0.docx to sites_default_files_2023-07_Voter%20ID%20Resource%20-%20Voter%20Booklet%20Easy%20Read%20English%20-%20Generic_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_Tips-on-managing-the-media-at-the-count.doc to sites_default_files_word_doc_Tips-on-managing-the-media-at-the-count.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-06_SLG%20Guidance%20on%20the%20retention%20and%20inspection%20of%20election%20documents_0.doc to sites_default_files_2022-06_SLG%20Guidance%20on%20the%20retention%20and%20inspection%20of%20election%20documents_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_FAQs%20for%20postal%20vote%20rejection%20notices.doc to sites_default_files_2022-01_FAQs%20for%20postal%20vote%20rejection%20notices.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Proof%20checking%20factsheet.doc to sites_default_files_2022-01_Proof%20checking%20factsheet.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-07_MAY%20Nomination%20pack%20incl%20election%20agent%20notification%20form.doc to sites_default_files_2022-07_MAY%20Nomination%20pack%20incl%20election%20agent%20notification%20form.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_SLG%20Notice%20of%20poll%20and%20situation%20of%20polling%20stations.doc to sites_default_files_2021-11_SLG%20Notice%20of%20poll%20and%20situation%20of%20polling%20stations.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Nominations%20considerations%20for%202021%20polls%20-%20E.docx to sites_default_files_2021-07_Nominations%20considerations%20for%202021%20polls%20-%20E.pdf\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_SP-Notice-of-election-constituency_0.doc to sites_default_files_2021-02_SP-Notice-of-election-constituency_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_LGW%20election%20timetable%20Generic.docx to sites_default_files_2022-02_LGW%20election%20timetable%20Generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_Senedd%20Notice%20of%20election%20-%20region.doc to sites_default_files_2021-03_Senedd%20Notice%20of%20election%20-%20region.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_Got5_SocialMediaTemplatePosts.docx to sites_default_files_2021-11_Got5_SocialMediaTemplatePosts.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-02_LGE%20election%20timetable%204%20May%202023.docx to sites_default_files_2023-02_LGE%20election%20timetable%204%20May%202023.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_PCC%20Nomination%20pack.doc to sites_default_files_2022-01_PCC%20Nomination%20pack.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Declaration%20for%20the%20companion%20of%20a%20disabled%20voter%20Senedd_0.doc to sites_default_files_2021-02_Declaration%20for%20the%20companion%20of%20a%20disabled%20voter%20Senedd_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-07_Canvass%202023%20-%20England%20press%20release%20template.docx to sites_default_files_2023-07_Canvass%202023%20-%20England%20press%20release%20template.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-07_Secrecy%20requirements%20verification%20and%20count%20S66.doc to sites_default_files_2019-07_Secrecy%20requirements%20verification%20and%20count%20S66.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_Withdrawal%20form%20Parish.doc to sites_default_files_2022-01_Withdrawal%20form%20Parish.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_Senedd%20Nomination%20checklist%20regional.docx to sites_default_files_2021-08_Senedd%20Nomination%20checklist%20regional.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2019-11_Timetable%20for%20a%20UK%20Parliamentary%20general%20election%20in%20Northern%20Ireland%2012%20December_0.docx to sites_default_files_2019-11_Timetable%20for%20a%20UK%20Parliamentary%20general%20election%20in%20Northern%20Ireland%2012%20December_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_SP-Nomination-pack-region-individual.doc to sites_default_files_2021-02_SP-Nomination-pack-region-individual.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20PCC%20Secrecy%20-%20postal%20voting.doc to sites_default_files_2021-02_Senedd%20PCC%20Secrecy%20-%20postal%20voting.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_Canvass%202023%20-%20Scotland%20press%20release%20template.docx to sites_default_files_2023-05_Canvass%202023%20-%20Scotland%20press%20release%20template.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_SP%20notice%20of%20poll%20and%20situation%20of%20polling%20stations%20%20-%20constituency.doc to sites_default_files_2021-02_SP%20notice%20of%20poll%20and%20situation%20of%20polling%20stations%20%20-%20constituency.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Senedd%20notice%20of%20poll%20and%20situation%20of%20polling%20stations%20%20-%20region.doc to sites_default_files_2021-02_Senedd%20notice%20of%20poll%20and%20situation%20of%20polling%20stations%20%20-%20region.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Guidance%20on%20the%20retention%20and%20inspection%20of%20election%20documents%20LGW.docx to sites_default_files_2022-02_Guidance%20on%20the%20retention%20and%20inspection%20of%20election%20documents%20LGW.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-01_PFCC%20Nomination%20pack.doc to sites_default_files_2022-01_PFCC%20Nomination%20pack.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-02_Design%20criteria%20forms%20and%20notices%20generic.doc to sites_default_files_2021-02_Design%20criteria%20forms%20and%20notices%20generic.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_SP%20Nomination%20checklist%20-%20regional.docx to sites_default_files_2021-08_SP%20Nomination%20checklist%20-%20regional.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_SP-Secrecy-postal-voting.doc to sites_default_files_word_doc_SP-Secrecy-postal-voting.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Supporting%20candidates%20and%20agents%20-%20May%202021%20polls%20S.docx to sites_default_files_2021-07_Supporting%20candidates%20and%20agents%20-%20May%202021%20polls%20S.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2023-05_User%20Guide%20for%20Post%20poll%20data%20collection%20May%202023%20External_0.docx to sites_default_files_2023-05_User%20Guide%20for%20Post%20poll%20data%20collection%20May%202023%20External_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_GLA%20Secrecy%20-%20postal%20voting.doc to sites_default_files_2021-08_GLA%20Secrecy%20-%20postal%20voting.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-03_SP%20FAQs%20for%20frontline%20staff_0.doc to sites_default_files_2021-03_SP%20FAQs%20for%20frontline%20staff_0.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_temp-soa-accruals-rp.doc to sites_default_files_word_doc_temp-soa-accruals-rp.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-07_Verification%20and%20count%20considerations%20for%202021%20polls%20-%20S.docx to sites_default_files_2021-07_Verification%20and%20count%20considerations%20for%202021%20polls%20-%20S.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_word_doc_SP-Scenarios-for-polling-station-staff.doc to sites_default_files_word_doc_SP-Scenarios-for-polling-station-staff.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-08_Senedd%20PCC%20-%20Count%20Scripts%20Bilingual.docx to sites_default_files_2021-08_Senedd%20PCC%20-%20Count%20Scripts%20Bilingual.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2021-11_SLG%20Polling%20station%20scenarios.doc to sites_default_files_2021-11_SLG%20Polling%20station%20scenarios.pdf\n",
      "Converted /Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages/sites_default_files_2022-02_Nomination%20checklist%20for%20Returning%20Officers%20LGW%20-%20Community%20Council.doc to sites_default_files_2022-02_Nomination%20checklist%20for%20Returning%20Officers%20LGW%20-%20Community%20Council.pdf\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import subprocess\n",
    "import glob\n",
    "import shutil\n",
    "\n",
    "def convert_to_pdf(input_directory):\n",
    "    # List all .doc and .docx files in the input directory\n",
    "    doc_files = glob.glob(os.path.join(input_directory, '*.doc*'))\n",
    "    \n",
    "    # Save current working directory\n",
    "    current_working_directory = os.getcwd()\n",
    "    \n",
    "    # Change working directory to input_directory\n",
    "    os.chdir(input_directory)\n",
    "    \n",
    "    for doc_file in doc_files:\n",
    "        # Construct the PDF file path in the input directory\n",
    "        base_name = os.path.basename(os.path.splitext(doc_file)[0])\n",
    "        pdf_file = f\"{base_name}.pdf\"\n",
    "        \n",
    "        try:\n",
    "            # Run unoconv to convert the doc/docx file to PDF\n",
    "            # Without specifying output directory, the file should be created in the current working directory\n",
    "            subprocess.run(['unoconv', '-f', 'pdf', os.path.basename(doc_file)], check=True)\n",
    "            print(f\"Converted {doc_file} to {pdf_file}\")\n",
    "        except subprocess.CalledProcessError as e:\n",
    "            print(f\"Failed to convert {doc_file} to PDF: {str(e)}\")\n",
    "        except Exception as e:\n",
    "            print(f\"An error occurred: {str(e)}\")\n",
    "    \n",
    "    # Change back to the original working directory\n",
    "    os.chdir(current_working_directory)\n",
    "\n",
    "# Use your directory path\n",
    "input_directory = '/Users/cardigan/Documents/GitHub/electoral_commission_chatbot/downloaded_pages'\n",
    "convert_to_pdf(input_directory)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ef2b24f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10 (XPython Raw)",
   "language": "python",
   "name": "xpython-raw"
  },
  "language_info": {
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}